From 9ea07b41da169f727a2eb7302adec4c724319522 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Wed, 14 Jan 2026 23:25:31 +0800 Subject: [PATCH] [1/N] Reorganize multimodal processing code (#32327) Signed-off-by: DarkLight1337 --- docs/api/README.md | 4 - docs/contributing/model/multimodal.md | 10 +- docs/design/mm_processing.md | 2 +- tests/multimodal/test_processing.py | 4 +- vllm/benchmarks/mm_processor.py | 2 +- vllm/model_executor/models/aria.py | 2 +- vllm/model_executor/models/audioflamingo3.py | 2 +- vllm/model_executor/models/aya_vision.py | 9 +- vllm/model_executor/models/bagel.py | 2 +- vllm/model_executor/models/blip2.py | 2 +- vllm/model_executor/models/chameleon.py | 2 +- vllm/model_executor/models/clip.py | 2 +- vllm/model_executor/models/cohere2_vision.py | 9 +- vllm/model_executor/models/deepseek_ocr.py | 2 +- vllm/model_executor/models/deepseek_vl2.py | 4 +- vllm/model_executor/models/ernie45_vl.py | 2 +- vllm/model_executor/models/fuyu.py | 2 +- vllm/model_executor/models/gemma3_mm.py | 4 +- vllm/model_executor/models/gemma3n_mm.py | 4 +- vllm/model_executor/models/glm4_1v.py | 2 +- vllm/model_executor/models/glm4v.py | 2 +- vllm/model_executor/models/glmasr.py | 2 +- vllm/model_executor/models/granite_speech.py | 2 +- vllm/model_executor/models/h2ovl.py | 2 +- vllm/model_executor/models/hunyuan_vision.py | 2 +- .../models/hyperclovax_vision.py | 2 +- vllm/model_executor/models/idefics3.py | 5 +- vllm/model_executor/models/interns1.py | 2 +- vllm/model_executor/models/internvl.py | 2 +- vllm/model_executor/models/isaac.py | 2 +- vllm/model_executor/models/kanana_v.py | 2 +- vllm/model_executor/models/keye.py | 2 +- vllm/model_executor/models/kimi_vl.py | 2 +- vllm/model_executor/models/lfm2_vl.py | 2 +- vllm/model_executor/models/lightonocr.py | 2 +- vllm/model_executor/models/llava.py | 2 +- .../model_executor/models/llava_next_video.py | 2 +- vllm/model_executor/models/midashenglm.py | 2 +- vllm/model_executor/models/minicpmv.py | 4 +- vllm/model_executor/models/mistral3.py | 2 +- vllm/model_executor/models/mllama4.py | 2 +- vllm/model_executor/models/molmo.py | 2 +- vllm/model_executor/models/molmo2.py | 2 +- .../model_executor/models/nano_nemotron_vl.py | 4 +- vllm/model_executor/models/nemotron_parse.py | 6 +- vllm/model_executor/models/ovis.py | 2 +- vllm/model_executor/models/ovis2_5.py | 2 +- vllm/model_executor/models/paddleocr_vl.py | 2 +- vllm/model_executor/models/paligemma.py | 2 +- vllm/model_executor/models/phi3v.py | 4 +- vllm/model_executor/models/phi4mm.py | 4 +- vllm/model_executor/models/pixtral.py | 4 +- .../models/qwen2_5_omni_thinker.py | 4 +- vllm/model_executor/models/qwen2_audio.py | 2 +- vllm/model_executor/models/qwen2_vl.py | 2 +- .../models/qwen3_omni_moe_thinker.py | 2 +- vllm/model_executor/models/qwen3_vl.py | 2 +- vllm/model_executor/models/qwen_vl.py | 2 +- vllm/model_executor/models/siglip.py | 2 +- vllm/model_executor/models/skyworkr1v.py | 2 +- vllm/model_executor/models/step3_vl.py | 2 +- vllm/model_executor/models/tarsier.py | 2 +- vllm/model_executor/models/terratorch.py | 2 +- .../models/transformers/multimodal.py | 7 +- vllm/model_executor/models/ultravox.py | 2 +- vllm/model_executor/models/voxtral.py | 4 +- .../models/voxtral_streaming.py | 4 +- vllm/model_executor/models/whisper.py | 2 +- vllm/multimodal/cache.py | 2 +- vllm/multimodal/inputs.py | 14 +- vllm/multimodal/processing/__init__.py | 27 + vllm/multimodal/processing/context.py | 558 +++++++++++++++++ .../dummy_inputs.py} | 12 +- .../processor.py} | 572 +----------------- vllm/multimodal/registry.py | 4 +- vllm/v1/engine/input_processor.py | 2 +- 76 files changed, 717 insertions(+), 670 deletions(-) create mode 100644 vllm/multimodal/processing/__init__.py create mode 100644 vllm/multimodal/processing/context.py rename vllm/multimodal/{profiling.py => processing/dummy_inputs.py} (97%) rename vllm/multimodal/{processing.py => processing/processor.py} (78%) diff --git a/docs/api/README.md b/docs/api/README.md index 14780c803c..da734ce1a1 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -82,10 +82,6 @@ Internal data structures. - [vllm.multimodal.processing][] -### Memory Profiling - -- [vllm.multimodal.profiling][] - ### Registry - [vllm.multimodal.registry][] diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md index c2ca199220..9e839a8e4c 100644 --- a/docs/contributing/model/multimodal.md +++ b/docs/contributing/model/multimodal.md @@ -116,12 +116,10 @@ def get_supported_mm_limits(self) -> Mapping[str, int | None]: ## 3. Specify dummy inputs -Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.profiling.BaseDummyInputsBuilder] to construct dummy inputs for -HF processing as well as memory profiling. +Then, inherit [BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] to construct dummy inputs for +HF processing. The processed outputs are also used for memory profiling. -### For memory profiling - -Override the abstract methods [get_dummy_text][vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs for memory profiling. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it. +Override the abstract methods [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] and [get_dummy_mm_data][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_mm_data] to construct dummy inputs. These dummy inputs should result in the worst-case memory usage of the model so that vLLM can reserve the correct amount of memory for it. Assuming that the memory usage increases with the number of tokens, the dummy inputs can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens. @@ -803,7 +801,7 @@ Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies ## 5. Register processor-related classes After you have defined [BaseProcessingInfo][vllm.multimodal.processing.BaseProcessingInfo] (Step 2), -[BaseDummyInputsBuilder][vllm.multimodal.profiling.BaseDummyInputsBuilder] (Step 3), +[BaseDummyInputsBuilder][vllm.multimodal.processing.BaseDummyInputsBuilder] (Step 3), and [BaseMultiModalProcessor][vllm.multimodal.processing.BaseMultiModalProcessor] (Step 4), decorate the model class with [MULTIMODAL_REGISTRY.register_processor][vllm.multimodal.registry.MultiModalRegistry.register_processor] to register them to the multi-modal registry: diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index ee56ac5b98..18d6c947e6 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -43,7 +43,7 @@ Moreover, since the tokenized text has not passed through the HF processor, we h ### Dummy text -We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_text]. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data. +We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data. ### Automatic prompt updating diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 7af0815d07..709f2f286b 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -9,8 +9,8 @@ import pytest from vllm.config import ModelConfig from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.processing import ( - InputProcessingContext, +from vllm.multimodal.processing.context import InputProcessingContext +from vllm.multimodal.processing.processor import ( PlaceholderFeaturesInfo, PromptIndexTargets, PromptInsertion, diff --git a/vllm/benchmarks/mm_processor.py b/vllm/benchmarks/mm_processor.py index 1e65a25539..a5512f1e55 100644 --- a/vllm/benchmarks/mm_processor.py +++ b/vllm/benchmarks/mm_processor.py @@ -24,7 +24,7 @@ import numpy as np from vllm.benchmarks.throughput import get_requests from vllm.engine.arg_utils import EngineArgs -from vllm.multimodal.processing import ( +from vllm.multimodal.processing.context import ( get_timing_stats_from_engine_client, ) from vllm.utils.gc_utils import freeze_gc_heap diff --git a/vllm/model_executor/models/aria.py b/vllm/model_executor/models/aria.py index c7f44762f3..b9a103ea90 100644 --- a/vllm/model_executor/models/aria.py +++ b/vllm/model_executor/models/aria.py @@ -30,12 +30,12 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index 3609cc26a4..f0e9f3e50b 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -47,13 +47,13 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/aya_vision.py b/vllm/model_executor/models/aya_vision.py index ee9e210a32..6b203cb4e5 100644 --- a/vllm/model_executor/models/aya_vision.py +++ b/vllm/model_executor/models/aya_vision.py @@ -18,17 +18,20 @@ from transformers.models.got_ocr2.image_processing_got_ocr2 import ( from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems +from vllm.multimodal.inputs import ( + MultiModalDataDict, + MultiModalFieldConfig, + MultiModalKwargsItems, +) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, - MultiModalFieldConfig, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/bagel.py b/vllm/model_executor/models/bagel.py index 08bb13e952..00124aa4ef 100644 --- a/vllm/model_executor/models/bagel.py +++ b/vllm/model_executor/models/bagel.py @@ -30,11 +30,11 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.bagel import BagelProcessor from vllm.utils.tensor_schema import TensorSchema diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index 2bd1dd1aef..14fd44ddb0 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -25,13 +25,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptIndexTargets, PromptInsertion, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/chameleon.py b/vllm/model_executor/models/chameleon.py index 176c5cd14c..5b8ecc0b6c 100644 --- a/vllm/model_executor/models/chameleon.py +++ b/vllm/model_executor/models/chameleon.py @@ -49,13 +49,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 1a4811ef8f..d49f8aa5ec 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -41,13 +41,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptIndexTargets, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index 07dc7a01dc..024fd390a0 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -27,17 +27,20 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.awq import AWQConfig from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import MultiModalDataDict, MultiModalKwargsItems +from vllm.multimodal.inputs import ( + MultiModalDataDict, + MultiModalFieldConfig, + MultiModalKwargsItems, +) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, - MultiModalFieldConfig, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/deepseek_ocr.py b/vllm/model_executor/models/deepseek_ocr.py index f396897bfa..f88efd6cba 100644 --- a/vllm/model_executor/models/deepseek_ocr.py +++ b/vllm/model_executor/models/deepseek_ocr.py @@ -39,12 +39,12 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sampling_params import SamplingParams from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config diff --git a/vllm/model_executor/models/deepseek_vl2.py b/vllm/model_executor/models/deepseek_vl2.py index 9f8faf9ed9..e5c6ab8bd8 100644 --- a/vllm/model_executor/models/deepseek_vl2.py +++ b/vllm/model_executor/models/deepseek_vl2.py @@ -32,14 +32,14 @@ from vllm.multimodal.parse import ( ImageSize, MultiModalDataItems, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, MultiModalProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config from vllm.transformers_utils.configs.deepseek_vl2 import ( diff --git a/vllm/model_executor/models/ernie45_vl.py b/vllm/model_executor/models/ernie45_vl.py index a382cb5b61..79cb1c5086 100644 --- a/vllm/model_executor/models/ernie45_vl.py +++ b/vllm/model_executor/models/ernie45_vl.py @@ -65,12 +65,12 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.v1.attention.backends.registry import AttentionBackendEnum diff --git a/vllm/model_executor/models/fuyu.py b/vllm/model_executor/models/fuyu.py index 8a7a3dd771..876e2645b0 100644 --- a/vllm/model_executor/models/fuyu.py +++ b/vllm/model_executor/models/fuyu.py @@ -38,13 +38,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 45dfacd944..9c986f182b 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -21,7 +21,8 @@ from vllm.multimodal.inputs import ( MultiModalKwargsItems, ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, MultiModalPromptUpdates, @@ -32,7 +33,6 @@ from vllm.multimodal.processing import ( PromptUpdateDetails, replace_token_matches, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index acb0d73990..14e3204dce 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -41,7 +41,8 @@ from vllm.multimodal.parse import ( MultiModalDataItems, MultiModalDataParser, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, MultiModalPromptUpdates, @@ -52,7 +53,6 @@ from vllm.multimodal.processing import ( PromptUpdateDetails, replace_token_matches, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 05257bd1ea..7cc55cf102 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -79,13 +79,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.v1.attention.backends.registry import AttentionBackendEnum diff --git a/vllm/model_executor/models/glm4v.py b/vllm/model_executor/models/glm4v.py index 297237fd19..615731bdad 100644 --- a/vllm/model_executor/models/glm4v.py +++ b/vllm/model_executor/models/glm4v.py @@ -43,12 +43,12 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs import ChatGLMConfig from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index a3b9a12219..ac5ec84def 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -39,13 +39,13 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config from vllm.transformers_utils.processor import cached_processor_from_config diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index 321704f0f9..dd0a00b56c 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -52,12 +52,12 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config from vllm.transformers_utils.processor import cached_processor_from_config diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index 6276c3d675..90b495e0d9 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -22,7 +22,7 @@ from vllm.multimodal.parse import ( ImageProcessorItems, MultiModalDataItems, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing.processor import ( MultiModalProcessingInfo, PromptReplacement, PromptUpdate, diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py index 9afb86a89f..82ea7d1ecf 100644 --- a/vllm/model_executor/models/hunyuan_vision.py +++ b/vllm/model_executor/models/hunyuan_vision.py @@ -66,12 +66,12 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.hunyuan_vl import ( HunYuanVLConfig, diff --git a/vllm/model_executor/models/hyperclovax_vision.py b/vllm/model_executor/models/hyperclovax_vision.py index f5226baba5..7bc9691c16 100644 --- a/vllm/model_executor/models/hyperclovax_vision.py +++ b/vllm/model_executor/models/hyperclovax_vision.py @@ -29,13 +29,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, InputProcessingContext, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index 459043e91d..9e8be5ca0c 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -42,16 +42,15 @@ from vllm.multimodal.inputs import ( MultiModalFieldConfig, MultiModalKwargsItems, ) -from vllm.multimodal.parse import ImageProcessorItems, ImageSize +from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, - MultiModalDataItems, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/interns1.py b/vllm/model_executor/models/interns1.py index 18985cefbf..4e01db279f 100644 --- a/vllm/model_executor/models/interns1.py +++ b/vllm/model_executor/models/interns1.py @@ -39,13 +39,13 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import cached_video_processor_from_config from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index 048bc49ea6..4fdf797cf3 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -41,13 +41,13 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/isaac.py b/vllm/model_executor/models/isaac.py index 6d331b95f9..377ed1e6f6 100644 --- a/vllm/model_executor/models/isaac.py +++ b/vllm/model_executor/models/isaac.py @@ -55,13 +55,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import get_tokenizer from vllm.tokenizers.hf import get_cached_tokenizer diff --git a/vllm/model_executor/models/kanana_v.py b/vllm/model_executor/models/kanana_v.py index 5e1667c1e4..3bc2031050 100644 --- a/vllm/model_executor/models/kanana_v.py +++ b/vllm/model_executor/models/kanana_v.py @@ -28,12 +28,12 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.import_utils import resolve_obj_by_qualname from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 8e6b664259..d1665efbdd 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -56,12 +56,12 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/kimi_vl.py b/vllm/model_executor/models/kimi_vl.py index b5436ca1af..2e88b97a57 100644 --- a/vllm/model_executor/models/kimi_vl.py +++ b/vllm/model_executor/models/kimi_vl.py @@ -82,12 +82,12 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs import KimiVLConfig, MoonViTConfig from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index d87b23d00c..d4671093ff 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -34,12 +34,12 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/lightonocr.py b/vllm/model_executor/models/lightonocr.py index 27ec12a8f1..897fbfdadf 100644 --- a/vllm/model_executor/models/lightonocr.py +++ b/vllm/model_executor/models/lightonocr.py @@ -31,12 +31,12 @@ from vllm.multimodal.cache import BaseMultiModalProcessorCache from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder _I = TypeVar("_I", bound=Mistral3ProcessingInfo) diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index ba54623d9d..44780b4ca2 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -39,6 +39,7 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, InputProcessingContext, @@ -46,7 +47,6 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/llava_next_video.py b/vllm/model_executor/models/llava_next_video.py index b146a144ea..3411abe1ab 100644 --- a/vllm/model_executor/models/llava_next_video.py +++ b/vllm/model_executor/models/llava_next_video.py @@ -26,12 +26,12 @@ from vllm.multimodal.parse import ( VideoProcessorItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.collection_utils import is_list_of from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/midashenglm.py b/vllm/model_executor/models/midashenglm.py index 2d506978d2..9b243f832b 100644 --- a/vllm/model_executor/models/midashenglm.py +++ b/vllm/model_executor/models/midashenglm.py @@ -54,13 +54,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.midashenglm import DashengConfig from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/minicpmv.py b/vllm/model_executor/models/minicpmv.py index 930ff737bc..499d247bcd 100644 --- a/vllm/model_executor/models/minicpmv.py +++ b/vllm/model_executor/models/minicpmv.py @@ -71,7 +71,8 @@ from vllm.multimodal.parse import ( VideoItem, VideoProcessorItems, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, @@ -80,7 +81,6 @@ from vllm.multimodal.processing import ( ResolvedPromptUpdate, _seq2text, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.utils.collection_utils import flatten_2d_lists diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index d3617fd4be..6c4418aa61 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -31,6 +31,7 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, InputProcessingContext, @@ -38,7 +39,6 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 13f79c91fe..190cc900c9 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -58,6 +58,7 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, InputProcessingContext, @@ -65,7 +66,6 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index bdfa6178b4..e30bc6889a 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -54,6 +54,7 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptIndexTargets, @@ -61,7 +62,6 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 33ee1d3436..bf4a298a75 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -67,13 +67,13 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.math_utils import round_down from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index a88496eca9..d13e8e4cb1 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -62,7 +62,8 @@ from vllm.multimodal.parse import ( MultiModalDataItems, MultiModalDataParser, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, @@ -70,7 +71,6 @@ from vllm.multimodal.processing import ( PromptUpdateDetails, _seq2tokens, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config from vllm.transformers_utils.configs.radio import RadioConfig diff --git a/vllm/model_executor/models/nemotron_parse.py b/vllm/model_executor/models/nemotron_parse.py index 83aba646ab..505ba35c63 100644 --- a/vllm/model_executor/models/nemotron_parse.py +++ b/vllm/model_executor/models/nemotron_parse.py @@ -53,14 +53,14 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseProcessingInfo, EncDecMultiModalProcessor, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.transformers_utils.configs.radio import RadioConfig -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.v1.attention.backend import AttentionType @@ -558,7 +558,7 @@ class NemotronParseProcessor: def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, **kwargs, ) -> None: super().__init__() diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index 0691bbc615..fda8489162 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -48,11 +48,11 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.ovis import OvisProcessor from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 945138b597..db59cb38ec 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -30,11 +30,11 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index 530974f7fa..5550734305 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -63,12 +63,12 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.v1.attention.backends.registry import AttentionBackendEnum diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py index 8671bbd5c7..40c43827b3 100644 --- a/vllm/model_executor/models/paligemma.py +++ b/vllm/model_executor/models/paligemma.py @@ -24,6 +24,7 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptIndexTargets, @@ -31,7 +32,6 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/phi3v.py b/vllm/model_executor/models/phi3v.py index 75823ec582..3de269309d 100644 --- a/vllm/model_executor/models/phi3v.py +++ b/vllm/model_executor/models/phi3v.py @@ -45,7 +45,8 @@ from vllm.multimodal.parse import ( ImageSize, MultiModalDataItems, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, MultiModalPromptUpdates, @@ -54,7 +55,6 @@ from vllm.multimodal.processing import ( PromptUpdate, ResolvedPromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/phi4mm.py b/vllm/model_executor/models/phi4mm.py index c58abefc85..2839c93049 100644 --- a/vllm/model_executor/models/phi4mm.py +++ b/vllm/model_executor/models/phi4mm.py @@ -40,14 +40,14 @@ from vllm.multimodal.parse import ( MultiModalDataItems, MultiModalDataParser, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ResolvedPromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index f57047bfa4..e60ab965ea 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -48,7 +48,8 @@ from vllm.multimodal.inputs import ( NestedTensors, ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, MultiModalProcessingInfo, @@ -56,7 +57,6 @@ from vllm.multimodal.processing import ( PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 744c21b54f..f2ee88b0a4 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -78,14 +78,14 @@ from vllm.multimodal.parse import ( MultiModalDataItems, MultiModalDataParser, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, MultiModalPromptUpdates, PlaceholderFeaturesInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/qwen2_audio.py b/vllm/model_executor/models/qwen2_audio.py index bc159b9273..2defffe4c0 100644 --- a/vllm/model_executor/models/qwen2_audio.py +++ b/vllm/model_executor/models/qwen2_audio.py @@ -54,13 +54,13 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 226d94c944..4f8e694d75 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -80,12 +80,12 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 0d13c3fe35..dee3cfa529 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -71,7 +71,7 @@ from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItems from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems -from vllm.multimodal.processing import ( +from vllm.multimodal.processing.processor import ( MultiModalPromptUpdates, PlaceholderFeaturesInfo, PromptReplacement, diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 279d280677..56c3db49ed 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -83,12 +83,12 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.collection_utils import is_list_of from vllm.v1.attention.backends.registry import AttentionBackendEnum diff --git a/vllm/model_executor/models/qwen_vl.py b/vllm/model_executor/models/qwen_vl.py index df0733de98..960e2606e5 100644 --- a/vllm/model_executor/models/qwen_vl.py +++ b/vllm/model_executor/models/qwen_vl.py @@ -42,13 +42,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 1bda00653b..37bb568dd5 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -46,13 +46,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptIndexTargets, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/skyworkr1v.py b/vllm/model_executor/models/skyworkr1v.py index f95fbffc1d..ec676805a6 100644 --- a/vllm/model_executor/models/skyworkr1v.py +++ b/vllm/model_executor/models/skyworkr1v.py @@ -39,13 +39,13 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/step3_vl.py b/vllm/model_executor/models/step3_vl.py index 771e5974ae..0983f24ab5 100644 --- a/vllm/model_executor/models/step3_vl.py +++ b/vllm/model_executor/models/step3_vl.py @@ -35,13 +35,13 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, PromptUpdateDetails, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.configs import Step3VisionEncoderConfig diff --git a/vllm/model_executor/models/tarsier.py b/vllm/model_executor/models/tarsier.py index dcfd432721..9f9bb73625 100644 --- a/vllm/model_executor/models/tarsier.py +++ b/vllm/model_executor/models/tarsier.py @@ -34,13 +34,13 @@ from vllm.multimodal.parse import ( MultiModalDataItems, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, InputProcessingContext, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/terratorch.py b/vllm/model_executor/models/terratorch.py index c97af0db5e..04b7069097 100644 --- a/vllm/model_executor/models/terratorch.py +++ b/vllm/model_executor/models/terratorch.py @@ -56,11 +56,11 @@ from vllm.multimodal.parse import ( MultiModalDataParser, ) from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from .interfaces import IsAttentionFree, MultiModalEmbeddings, SupportsMultiModal diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index fcf9a0d077..68c90992e1 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -35,8 +35,11 @@ from vllm.multimodal.inputs import ( PlaceholderRange, ) from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems -from vllm.multimodal.processing import BaseMultiModalProcessor, BaseProcessingInfo -from vllm.multimodal.profiling import BaseDummyInputsBuilder +from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, + BaseMultiModalProcessor, + BaseProcessingInfo, +) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index 7fb46979ba..f161fb5f96 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -36,12 +36,12 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.ultravox import UltravoxConfig from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 63b26c7890..ec3c1edc19 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -47,14 +47,14 @@ from vllm.multimodal.parse import ( MultiModalDataItems, MultiModalDataParser, ) -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder, ProcessorInputs +from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, MultiModalProcessingInfo, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config from vllm.tokenizers.mistral import MistralTokenizer diff --git a/vllm/model_executor/models/voxtral_streaming.py b/vllm/model_executor/models/voxtral_streaming.py index a89a0eedd8..730576a401 100644 --- a/vllm/model_executor/models/voxtral_streaming.py +++ b/vllm/model_executor/models/voxtral_streaming.py @@ -30,11 +30,11 @@ from vllm.multimodal.inputs import ( MultiModalKwargsOptionalItems, ) from vllm.multimodal.parse import MultiModalDataItems -from vllm.multimodal.processing import ( +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( MultiModalPromptUpdates, PlaceholderFeaturesInfo, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index 6513088575..379a61ceac 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -49,12 +49,12 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, BaseProcessingInfo, EncDecMultiModalProcessor, PromptReplacement, PromptUpdate, ) -from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.jsontree import json_map_leaves from vllm.utils.tensor_schema import TensorSchema, TensorShape diff --git a/vllm/multimodal/cache.py b/vllm/multimodal/cache.py index 41397a26ef..7f97ae3595 100644 --- a/vllm/multimodal/cache.py +++ b/vllm/multimodal/cache.py @@ -34,7 +34,7 @@ from .inputs import ( if TYPE_CHECKING: from vllm.config import ModelConfig, VllmConfig - from .processing import ResolvedPromptUpdate + from .processing.processor import ResolvedPromptUpdate from .registry import MultiModalRegistry logger = init_logger(__name__) diff --git a/vllm/multimodal/inputs.py b/vllm/multimodal/inputs.py index bd49d71923..1653fa0c23 100644 --- a/vllm/multimodal/inputs.py +++ b/vllm/multimodal/inputs.py @@ -33,8 +33,6 @@ if TYPE_CHECKING: from transformers.feature_extraction_utils import BatchFeature from .base import MediaWithBytes - from .processing import MultiModalHashes - else: torch = LazyLoader("torch", globals(), "torch") @@ -979,9 +977,15 @@ MultiModalKwargsOptionalItems: TypeAlias = ( ) +MultiModalHashes = dict[str, list[str]] +""" +A dictionary containing per-item hashes for each modality. +""" + + MultiModalPlaceholderDict: TypeAlias = Mapping[str, Sequence[PlaceholderRange]] """ -A dictionary containing placeholder ranges for each modality. +A dictionary containing per-item placeholder ranges for each modality. """ @@ -1001,10 +1005,10 @@ class MultiModalInputs(TypedDict): mm_kwargs: MultiModalKwargsOptionalItems """Keyword arguments to be directly passed to the model after batching.""" - mm_hashes: "MultiModalHashes" + mm_hashes: MultiModalHashes """The hashes of the multi-modal data.""" - mm_placeholders: "MultiModalPlaceholderDict" + mm_placeholders: MultiModalPlaceholderDict """ For each modality, information about the placeholder tokens in `prompt_token_ids`. diff --git a/vllm/multimodal/processing/__init__.py b/vllm/multimodal/processing/__init__.py new file mode 100644 index 0000000000..d248703afb --- /dev/null +++ b/vllm/multimodal/processing/__init__.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from .context import BaseProcessingInfo, InputProcessingContext +from .dummy_inputs import BaseDummyInputsBuilder, ProcessorInputs +from .processor import ( + BaseMultiModalProcessor, + EncDecMultiModalProcessor, + PromptIndexTargets, + PromptInsertion, + PromptReplacement, + PromptUpdate, + PromptUpdateDetails, +) + +__all__ = [ + "BaseProcessingInfo", + "InputProcessingContext", + "BaseDummyInputsBuilder", + "ProcessorInputs", + "BaseMultiModalProcessor", + "EncDecMultiModalProcessor", + "PromptUpdate", + "PromptIndexTargets", + "PromptUpdateDetails", + "PromptInsertion", + "PromptReplacement", +] diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py new file mode 100644 index 0000000000..8606a659c3 --- /dev/null +++ b/vllm/multimodal/processing/context.py @@ -0,0 +1,558 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import contextvars +import threading +import time +from abc import abstractmethod +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import ( + TYPE_CHECKING, + Any, + overload, +) + +import torch +from typing_extensions import TypeVar + +from vllm.logger import init_logger +from vllm.tokenizers import TokenizerLike +from vllm.transformers_utils.processor import cached_processor_from_config +from vllm.utils.func_utils import get_allowed_kwarg_only_overrides +from vllm.utils.jsontree import JSONTree, json_map_leaves + +if TYPE_CHECKING: + from transformers.configuration_utils import PretrainedConfig + from transformers.feature_extraction_utils import BatchFeature + from transformers.processing_utils import ProcessorMixin + + from vllm.config import ModelConfig, ObservabilityConfig +else: + PretrainedConfig = object + BatchFeature = object + ProcessorMixin = object + + ModelConfig = object + ObservabilityConfig = object + +logger = init_logger(__name__) + + +_request_id_context: contextvars.ContextVar[str | None] = contextvars.ContextVar( + "_request_id_context", default=None +) + + +def get_current_request_id() -> str | None: + """Get the current request_id from the context, if available.""" + return _request_id_context.get() + + +@contextmanager +def set_request_id(request_id: str) -> Generator[None, None, None]: + """Context manager to set the request_id for the current context.""" + token = _request_id_context.set(request_id) + try: + yield + finally: + _request_id_context.reset(token) + + +@dataclass +class MultiModalProcessorTimingStats: + """Per-request timing statistics for multimodal processor stages.""" + + hf_processor_time: float = 0.0 + """Time spent in HuggingFace processor calls (seconds).""" + + hashing_time: float = 0.0 + """Time spent computing multimodal item hashes (seconds).""" + + cache_lookup_time: float = 0.0 + """Time spent in cache lookups and merges (seconds).""" + + prompt_update_time: float = 0.0 + """Time spent applying prompt updates and finding placeholders (seconds).""" + + total_time: float = 0.0 + """Total processing time (seconds).""" + + def to_dict(self) -> dict[str, float]: + """Convert stats to a dictionary for JSON serialization.""" + return { + "hf_processor_time": self.hf_processor_time, + "hashing_time": self.hashing_time, + "cache_lookup_time": self.cache_lookup_time, + "prompt_update_time": self.prompt_update_time, + "total_time": self.total_time, + } + + +def get_timing_stats_from_engine_client( + engine_client: Any, +) -> dict[str, dict[str, float]]: + """ + Get all timing stats from the context associated with the engine client. + + Args: + engine_client: The engine client that has input_processor. + + Returns: + A dictionary mapping request_id to stats dict. + """ + try: + if not engine_client.vllm_config.observability_config.enable_mm_processor_stats: + return {} + except (AttributeError, RuntimeError): + return {} + + try: + input_processor = engine_client.input_processor + input_preprocessor = input_processor.input_preprocessor + + if hasattr(input_preprocessor, "_get_mm_processor"): + mm_processor = input_preprocessor._get_mm_processor() + if mm_processor is not None and hasattr(mm_processor, "info"): + ctx = mm_processor.info.ctx + return ctx.get_all_timing_stats() + except (AttributeError, RuntimeError): + pass + + return {} + + +@contextmanager +def timed_operation(ctx: "InputProcessingContext", stage_name: str): + """ + Context manager to time an operation using the context's timing stats. + + The request_id is automatically retrieved from the context variable, + so it doesn't need to be passed as a parameter. + + Args: + ctx: The InputProcessingContext containing the timing stats registry. + stage_name: Name of the stage being timed. + """ + request_id = get_current_request_id() + if ctx is None or request_id is None: + yield + return + + stats = ctx.get_timing_stats(request_id) + if stats is None: + yield + return + + start_time = time.perf_counter() + try: + yield + finally: + elapsed = time.perf_counter() - start_time + if stage_name == "hf_processor": + stats.hf_processor_time += elapsed + elif stage_name == "hashing": + stats.hashing_time += elapsed + elif stage_name == "cache_lookup": + stats.cache_lookup_time += elapsed + elif stage_name == "prompt_update": + stats.prompt_update_time += elapsed + stats.total_time += elapsed + + +_T = TypeVar("_T") +_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig) +_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin) + + +@dataclass(frozen=True) +class InputProcessingContext: + """ + Contains information about the model which may be used to + modify the inputs. + """ + + model_config: ModelConfig + """The configuration of the model.""" + + tokenizer: TokenizerLike | None + """The tokenizer used to tokenize the inputs.""" + + observability_config: "ObservabilityConfig | None" = field( + default=None, compare=False, repr=False + ) + """Configuration for observability features.""" + + timing_stats_registry: dict[str, MultiModalProcessorTimingStats] = field( + default_factory=dict, compare=False, repr=False + ) + """Registry for storing timing stats keyed by request_id.""" + + _timing_stats_registry_lock: threading.Lock = field( + default_factory=threading.Lock, compare=False, repr=False + ) + """Lock for thread-safe access to timing_stats_registry.""" + + def get_tokenizer(self) -> TokenizerLike: + if self.tokenizer is None: + raise ValueError( + "You cannot pass text prompts when `skip_tokenizer_init=True`" + ) + + return self.tokenizer + + @overload + def get_hf_config(self, /) -> PretrainedConfig: ... + + @overload + def get_hf_config( + self, + typ: type[_C] | tuple[type[_C], ...], + /, + ) -> _C: ... + + def get_hf_config( + self, + typ: type[Any] | tuple[type[Any], ...] | None = None, + /, + ) -> Any: + """ + Get the HuggingFace configuration + (`transformers.PretrainedConfig`) of the model, + additionally checking its type. + + Raises: + TypeError: If the configuration is not of the specified type. + """ + if typ is None: + from transformers.configuration_utils import PretrainedConfig + + typ = PretrainedConfig + + hf_config = self.model_config.hf_config + if not isinstance(hf_config, typ): + raise TypeError( + "Invalid type of HuggingFace config. " + f"Expected type: {typ}, but " + f"found type: {type(hf_config)}" + ) + + return hf_config + + def get_hf_image_processor_config(self) -> dict[str, Any]: + """ + Get the HuggingFace image processor configuration of the model. + """ + return self.model_config.hf_image_processor_config + + def get_mm_config(self): + """ + Get the multimodal config of the model. + + Raises: + RuntimeError: If the model is not a multimodal model. + """ + mm_config = self.model_config.multimodal_config + if mm_config is None: + raise RuntimeError("Not a multimodal model") + + return mm_config + + @overload + def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ... + + @overload + def get_hf_processor( + self, + typ: type[_P] | tuple[type[_P], ...], + /, + **kwargs: object, + ) -> _P: ... + + def get_hf_processor( + self, + typ: type[Any] | tuple[type[Any], ...] | None = None, + /, + **kwargs: object, + ) -> Any: + """ + Get the HuggingFace processor + (`transformers.ProcessorMixin`) of the model, + additionally checking its type. + + Raises: + TypeError: If the processor is not of the specified type. + """ + if typ is None: + from transformers.processing_utils import ProcessorMixin + + typ = ProcessorMixin + + from vllm.tokenizers.mistral import MistralTokenizer + + tokenizer = self.tokenizer + if isinstance(tokenizer, MistralTokenizer): + tokenizer = tokenizer.transformers_tokenizer + + return cached_processor_from_config( + self.model_config, + processor_cls=typ, + tokenizer=tokenizer, + **kwargs, + ) + + def init_processor( + self, + typ: type[_T], + /, + **kwargs: object, + ) -> _T: + """ + Initialize a HuggingFace-like processor class, merging the + keyword arguments with those in the model's configuration. + """ + mm_config = self.model_config.get_multimodal_config() + base_kwargs = mm_config.mm_processor_kwargs + if base_kwargs is None: + base_kwargs = {} + + merged_kwargs = {**base_kwargs, **kwargs} + + return typ(**merged_kwargs) + + def _postprocess_output( + self, + output: JSONTree, + ) -> JSONTree: + def _postprocess_one(x: object): + if isinstance(x, torch.Tensor): # noqa: SIM102 + # This mimics the behavior of transformers.BatchFeature + if x.is_floating_point(): + x = x.to(dtype=self.model_config.dtype) + + return x + + return json_map_leaves(_postprocess_one, output) + + def call_hf_processor( + self, + hf_processor: ProcessorMixin, + data: Mapping[str, object], + kwargs: Mapping[str, object] = {}, + *, + num_tries: int = 1, + max_tries: int = 5, + ) -> BatchFeature | JSONTree: + """ + Call `hf_processor` on the prompt `data` + (text, image, audio...) with configurable options `kwargs`. + """ + assert callable(hf_processor) + + mm_config = self.model_config.get_multimodal_config() + merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs) + + allowed_kwargs = get_allowed_kwarg_only_overrides( + hf_processor, + merged_kwargs, + requires_kw_only=False, + allow_var_kwargs=True, + ) + + try: + output = hf_processor(**data, **allowed_kwargs, return_tensors="pt") + except Exception as exc: + # See https://github.com/huggingface/tokenizers/issues/537 + if ( + isinstance(exc, RuntimeError) + and exc + and exc.args[0] == "Already borrowed" + and num_tries < max_tries + ): + logger.warning( + "Failed to acquire tokenizer in current thread. " + "Retrying (%d/%d)...", + num_tries, + max_tries, + ) + time.sleep(0.5) + return self.call_hf_processor( + hf_processor, + data, + kwargs, + num_tries=num_tries + 1, + max_tries=max_tries, + ) + + msg = ( + f"Failed to apply {type(hf_processor).__name__} " + f"on data={data} with kwargs={allowed_kwargs}" + ) + + raise ValueError(msg) from exc + + # this emulates output.to(dtype=self.model_config.dtype) + from transformers.feature_extraction_utils import BatchFeature + + if isinstance(output, BatchFeature): + output_ = self._postprocess_output(output.data) + return BatchFeature(output_) + + logger.warning_once( + "%s did not return `BatchFeature`. " + "Make sure to match the behaviour of `ProcessorMixin` when " + "implementing custom processors.", + type(hf_processor).__name__, + ) + + return self._postprocess_output(output) + + def get_timing_stats( + self, request_id: str + ) -> MultiModalProcessorTimingStats | None: + """ + Get timing stats for a request. + """ + if ( + self.observability_config is None + or not self.observability_config.enable_mm_processor_stats + ): + return None + with self._timing_stats_registry_lock: + return self.timing_stats_registry.get(request_id) + + def create_timing_stats(self, request_id: str) -> MultiModalProcessorTimingStats: + """ + Create and store timing stats in the registry for a request. + + This should be called at the start of processing for a request. + The stats object is created immediately and stored in the registry. + """ + if ( + self.observability_config is None + or not self.observability_config.enable_mm_processor_stats + ): + return MultiModalProcessorTimingStats() + + with self._timing_stats_registry_lock: + if request_id in self.timing_stats_registry: + raise ValueError( + f"Timing stats already exist for request_id: {request_id}" + ) + stats = MultiModalProcessorTimingStats() + self.timing_stats_registry[request_id] = stats + return stats + + def clear_timing_stats_registry(self) -> int: + """ + Clear all stats from the registry. Returns the number of stats cleared. + """ + if ( + self.observability_config is None + or not self.observability_config.enable_mm_processor_stats + ): + return 0 + with self._timing_stats_registry_lock: + count = len(self.timing_stats_registry) + self.timing_stats_registry.clear() + return count + + def get_all_timing_stats(self) -> dict[str, dict[str, float]]: + """ + Get all timing stats as a dictionary for API endpoints. + """ + if ( + self.observability_config is None + or not self.observability_config.enable_mm_processor_stats + ): + return {} + with self._timing_stats_registry_lock: + return { + rid: stats.to_dict() + for rid, stats in self.timing_stats_registry.items() + } + + +class BaseProcessingInfo: + """Base class to provide the information necessary for data processing.""" + + def __init__(self, ctx: InputProcessingContext) -> None: + super().__init__() + + self.ctx = ctx + + @property + def model_id(self) -> str: + return self.ctx.model_config.model + + def get_tokenizer(self) -> TokenizerLike: + return self.ctx.get_tokenizer() + + def get_hf_config(self) -> PretrainedConfig: + return self.ctx.get_hf_config() + + def get_hf_processor(self, **kwargs: object) -> ProcessorMixin: + """ + Subclasses can override this method to handle + specific kwargs from model config or user inputs. + """ + return self.ctx.get_hf_processor(**kwargs) + + @property + def skip_prompt_length_check(self) -> bool: + return False + + @abstractmethod + def get_supported_mm_limits(self) -> Mapping[str, int | None]: + """ + Return the maximum supported number of items for each modality. + + A value of `None` means unlimited number of items. + + Omitting a modality from the returned dictionary means that + it is not supported at all. + """ + raise NotImplementedError + + def get_allowed_mm_limits(self) -> Mapping[str, int]: + """Return the maximum allowed number of items for each modality.""" + supported_mm_limits = self.get_supported_mm_limits() + mm_config = self.ctx.get_mm_config() + + allowed_limits = dict[str, int]() + for modality, supported_limit in supported_mm_limits.items(): + user_limit = mm_config.get_limit_per_prompt(modality) + + allowed_limits[modality] = ( + user_limit + if supported_limit is None + else min(user_limit, supported_limit) + ) + + return allowed_limits + + def get_mm_max_tokens_per_item( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> Mapping[str, int] | None: + """ + Return the maximum number of tokens per item of for each modality. + + When `None` (the default) is returned, vLLM will generate dummy inputs + (images/videos) at maximum possible sizes and process them to determine + the maximum token count per modality. + + This approach works but can be very slow for certain models (e.g., + Qwen2.5-VL), leading to very long startup time. For better performance, + each model can override this method to return pre-computed maximum token + counts, avoiding the need for dummy input generation and processing. + + Note: + The maximum number of tokens per item of each modality returned + from this function should respect the model's maximum sequence + length and the maximum number of items of each modality allowed, + and agree with dummy inputs (images/videos) at maximum possible + sizes. + """ + return None diff --git a/vllm/multimodal/profiling.py b/vllm/multimodal/processing/dummy_inputs.py similarity index 97% rename from vllm/multimodal/profiling.py rename to vllm/multimodal/processing/dummy_inputs.py index 56f0dfc77e..05e76fe7a3 100644 --- a/vllm/multimodal/profiling.py +++ b/vllm/multimodal/processing/dummy_inputs.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Generic +from typing import Generic, TypeVar import numpy as np import numpy.typing as npt @@ -17,14 +17,10 @@ from vllm.config.multimodal import ( ) from vllm.logger import init_logger -from .inputs import MultiModalDataDict +from ..inputs import MultiModalDataDict +from .context import BaseProcessingInfo -if TYPE_CHECKING: - from .processing import _I -else: - from typing import TypeVar - - _I = TypeVar("_I") +_I = TypeVar("_I", bound=BaseProcessingInfo) logger = init_logger(__name__) diff --git a/vllm/multimodal/processing.py b/vllm/multimodal/processing/processor.py similarity index 78% rename from vllm/multimodal/processing.py rename to vllm/multimodal/processing/processor.py index f32b938afc..3aee5ed8bf 100644 --- a/vllm/multimodal/processing.py +++ b/vllm/multimodal/processing/processor.py @@ -1,24 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import contextvars -import threading -import time from abc import ABC, abstractmethod from collections import defaultdict from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence -from contextlib import contextmanager from dataclasses import dataclass, field, replace from enum import Enum from functools import lru_cache from typing import ( TYPE_CHECKING, - Any, Generic, NamedTuple, Protocol, TypeAlias, cast, - overload, ) import regex as re @@ -27,16 +21,14 @@ from typing_extensions import TypeVar, assert_never from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike -from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.collection_utils import flatten_2d_lists, full_groupby -from vllm.utils.func_utils import get_allowed_kwarg_only_overrides -from vllm.utils.jsontree import JSONTree, json_map_leaves -from .hasher import MultiModalHasher -from .inputs import ( +from ..hasher import MultiModalHasher +from ..inputs import ( MultiModalDataDict, MultiModalEncDecInputs, MultiModalFieldConfig, + MultiModalHashes, MultiModalInputs, MultiModalKwargsItem, MultiModalKwargsItems, @@ -44,29 +36,21 @@ from .inputs import ( MultiModalUUIDDict, PlaceholderRange, ) -from .parse import ( +from ..parse import ( DictEmbeddingItems, EmbeddingItems, MultiModalDataItems, MultiModalDataParser, ) -from .profiling import BaseDummyInputsBuilder +from .context import BaseProcessingInfo, get_current_request_id, timed_operation +from .dummy_inputs import BaseDummyInputsBuilder if TYPE_CHECKING: - from transformers.configuration_utils import PretrainedConfig from transformers.feature_extraction_utils import BatchFeature - from transformers.processing_utils import ProcessorMixin - from vllm.config import ModelConfig, ObservabilityConfig - - from .cache import BaseMultiModalProcessorCache + from ..cache import BaseMultiModalProcessorCache else: - PretrainedConfig = object BatchFeature = object - ProcessorMixin = object - - ModelConfig = object - ObservabilityConfig = object BaseMultiModalProcessorCache = object @@ -74,126 +58,6 @@ logger = init_logger(__name__) _S = TypeVar("_S", str, list[int]) -_request_id_context: contextvars.ContextVar[str | None] = contextvars.ContextVar( - "_request_id_context", default=None -) - - -def get_current_request_id() -> str | None: - """Get the current request_id from the context, if available.""" - return _request_id_context.get() - - -@contextmanager -def set_request_id(request_id: str) -> Generator[None, None, None]: - """Context manager to set the request_id for the current context.""" - token = _request_id_context.set(request_id) - try: - yield - finally: - _request_id_context.reset(token) - - -@dataclass -class MultiModalProcessorTimingStats: - """Per-request timing statistics for multimodal processor stages.""" - - hf_processor_time: float = 0.0 - """Time spent in HuggingFace processor calls (seconds).""" - - hashing_time: float = 0.0 - """Time spent computing multimodal item hashes (seconds).""" - - cache_lookup_time: float = 0.0 - """Time spent in cache lookups and merges (seconds).""" - - prompt_update_time: float = 0.0 - """Time spent applying prompt updates and finding placeholders (seconds).""" - - total_time: float = 0.0 - """Total processing time (seconds).""" - - def to_dict(self) -> dict[str, float]: - """Convert stats to a dictionary for JSON serialization.""" - return { - "hf_processor_time": self.hf_processor_time, - "hashing_time": self.hashing_time, - "cache_lookup_time": self.cache_lookup_time, - "prompt_update_time": self.prompt_update_time, - "total_time": self.total_time, - } - - -def get_timing_stats_from_engine_client( - engine_client: Any, -) -> dict[str, dict[str, float]]: - """ - Get all timing stats from the context associated with the engine client. - - Args: - engine_client: The engine client that has input_processor. - - Returns: - A dictionary mapping request_id to stats dict. - """ - try: - if not engine_client.vllm_config.observability_config.enable_mm_processor_stats: - return {} - except (AttributeError, RuntimeError): - return {} - - try: - input_processor = engine_client.input_processor - input_preprocessor = input_processor.input_preprocessor - - if hasattr(input_preprocessor, "_get_mm_processor"): - mm_processor = input_preprocessor._get_mm_processor() - if mm_processor is not None and hasattr(mm_processor, "info"): - ctx = mm_processor.info.ctx - return ctx.get_all_timing_stats() - except (AttributeError, RuntimeError): - pass - - return {} - - -@contextmanager -def _timed_operation(ctx: "InputProcessingContext", stage_name: str): - """ - Context manager to time an operation using the context's timing stats. - - The request_id is automatically retrieved from the context variable, - so it doesn't need to be passed as a parameter. - - Args: - ctx: The InputProcessingContext containing the timing stats registry. - stage_name: Name of the stage being timed. - """ - request_id = get_current_request_id() - if ctx is None or request_id is None: - yield - return - - stats = ctx.get_timing_stats(request_id) - if stats is None: - yield - return - - start_time = time.perf_counter() - try: - yield - finally: - elapsed = time.perf_counter() - start_time - if stage_name == "hf_processor": - stats.hf_processor_time += elapsed - elif stage_name == "hashing": - stats.hashing_time += elapsed - elif stage_name == "cache_lookup": - stats.cache_lookup_time += elapsed - elif stage_name == "prompt_update": - stats.prompt_update_time += elapsed - stats.total_time += elapsed - PromptSeq: TypeAlias = str | list[int] """A token sequence (list of token IDs) or text.""" @@ -1073,412 +937,6 @@ def find_mm_placeholders( return dict(full_groupby_modality(it)) -_T = TypeVar("_T") -_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig) -_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin) - - -@dataclass(frozen=True) -class InputProcessingContext: - """ - Contains information about the model which may be used to - modify the inputs. - """ - - model_config: ModelConfig - """The configuration of the model.""" - - tokenizer: TokenizerLike | None - """The tokenizer used to tokenize the inputs.""" - - observability_config: "ObservabilityConfig | None" = field( - default=None, compare=False, repr=False - ) - """Configuration for observability features.""" - - timing_stats_registry: dict[str, MultiModalProcessorTimingStats] = field( - default_factory=dict, compare=False, repr=False - ) - """Registry for storing timing stats keyed by request_id.""" - - _timing_stats_registry_lock: threading.Lock = field( - default_factory=threading.Lock, compare=False, repr=False - ) - """Lock for thread-safe access to timing_stats_registry.""" - - def get_tokenizer(self) -> TokenizerLike: - if self.tokenizer is None: - raise ValueError( - "You cannot pass text prompts when `skip_tokenizer_init=True`" - ) - - return self.tokenizer - - @overload - def get_hf_config(self, /) -> PretrainedConfig: ... - - @overload - def get_hf_config( - self, - typ: type[_C] | tuple[type[_C], ...], - /, - ) -> _C: ... - - def get_hf_config( - self, - typ: type[Any] | tuple[type[Any], ...] | None = None, - /, - ) -> Any: - """ - Get the HuggingFace configuration - (`transformers.PretrainedConfig`) of the model, - additionally checking its type. - - Raises: - TypeError: If the configuration is not of the specified type. - """ - if typ is None: - from transformers.configuration_utils import PretrainedConfig - - typ = PretrainedConfig - - hf_config = self.model_config.hf_config - if not isinstance(hf_config, typ): - raise TypeError( - "Invalid type of HuggingFace config. " - f"Expected type: {typ}, but " - f"found type: {type(hf_config)}" - ) - - return hf_config - - def get_hf_image_processor_config(self) -> dict[str, Any]: - """ - Get the HuggingFace image processor configuration of the model. - """ - return self.model_config.hf_image_processor_config - - def get_mm_config(self): - """ - Get the multimodal config of the model. - - Raises: - RuntimeError: If the model is not a multimodal model. - """ - mm_config = self.model_config.multimodal_config - if mm_config is None: - raise RuntimeError("Not a multimodal model") - - return mm_config - - @overload - def get_hf_processor(self, /, **kwargs: object) -> ProcessorMixin: ... - - @overload - def get_hf_processor( - self, - typ: type[_P] | tuple[type[_P], ...], - /, - **kwargs: object, - ) -> _P: ... - - def get_hf_processor( - self, - typ: type[Any] | tuple[type[Any], ...] | None = None, - /, - **kwargs: object, - ) -> Any: - """ - Get the HuggingFace processor - (`transformers.ProcessorMixin`) of the model, - additionally checking its type. - - Raises: - TypeError: If the processor is not of the specified type. - """ - if typ is None: - from transformers.processing_utils import ProcessorMixin - - typ = ProcessorMixin - - from vllm.tokenizers.mistral import MistralTokenizer - - tokenizer = self.tokenizer - if isinstance(tokenizer, MistralTokenizer): - tokenizer = tokenizer.transformers_tokenizer - - return cached_processor_from_config( - self.model_config, - processor_cls=typ, - tokenizer=tokenizer, - **kwargs, - ) - - def init_processor( - self, - typ: type[_T], - /, - **kwargs: object, - ) -> _T: - """ - Initialize a HuggingFace-like processor class, merging the - keyword arguments with those in the model's configuration. - """ - mm_config = self.model_config.get_multimodal_config() - base_kwargs = mm_config.mm_processor_kwargs - if base_kwargs is None: - base_kwargs = {} - - merged_kwargs = {**base_kwargs, **kwargs} - - return typ(**merged_kwargs) - - def _postprocess_output( - self, - output: JSONTree, - ) -> JSONTree: - def _postprocess_one(x: object): - if isinstance(x, torch.Tensor): # noqa: SIM102 - # This mimics the behavior of transformers.BatchFeature - if x.is_floating_point(): - x = x.to(dtype=self.model_config.dtype) - - return x - - return json_map_leaves(_postprocess_one, output) - - def call_hf_processor( - self, - hf_processor: ProcessorMixin, - data: Mapping[str, object], - kwargs: Mapping[str, object] = {}, - *, - num_tries: int = 1, - max_tries: int = 5, - ) -> BatchFeature | JSONTree: - """ - Call `hf_processor` on the prompt `data` - (text, image, audio...) with configurable options `kwargs`. - """ - assert callable(hf_processor) - - mm_config = self.model_config.get_multimodal_config() - merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs) - - allowed_kwargs = get_allowed_kwarg_only_overrides( - hf_processor, - merged_kwargs, - requires_kw_only=False, - allow_var_kwargs=True, - ) - - try: - output = hf_processor(**data, **allowed_kwargs, return_tensors="pt") - except Exception as exc: - # See https://github.com/huggingface/tokenizers/issues/537 - if ( - isinstance(exc, RuntimeError) - and exc - and exc.args[0] == "Already borrowed" - and num_tries < max_tries - ): - logger.warning( - "Failed to acquire tokenizer in current thread. " - "Retrying (%d/%d)...", - num_tries, - max_tries, - ) - time.sleep(0.5) - return self.call_hf_processor( - hf_processor, - data, - kwargs, - num_tries=num_tries + 1, - max_tries=max_tries, - ) - - msg = ( - f"Failed to apply {type(hf_processor).__name__} " - f"on data={data} with kwargs={allowed_kwargs}" - ) - - raise ValueError(msg) from exc - - # this emulates output.to(dtype=self.model_config.dtype) - from transformers.feature_extraction_utils import BatchFeature - - if isinstance(output, BatchFeature): - output_ = self._postprocess_output(output.data) - return BatchFeature(output_) - - logger.warning_once( - "%s did not return `BatchFeature`. " - "Make sure to match the behaviour of `ProcessorMixin` when " - "implementing custom processors.", - type(hf_processor).__name__, - ) - - return self._postprocess_output(output) - - def get_timing_stats( - self, request_id: str - ) -> MultiModalProcessorTimingStats | None: - """ - Get timing stats for a request. - """ - if ( - self.observability_config is None - or not self.observability_config.enable_mm_processor_stats - ): - return None - with self._timing_stats_registry_lock: - return self.timing_stats_registry.get(request_id) - - def create_timing_stats(self, request_id: str) -> MultiModalProcessorTimingStats: - """ - Create and store timing stats in the registry for a request. - - This should be called at the start of processing for a request. - The stats object is created immediately and stored in the registry. - """ - if ( - self.observability_config is None - or not self.observability_config.enable_mm_processor_stats - ): - return MultiModalProcessorTimingStats() - - with self._timing_stats_registry_lock: - if request_id in self.timing_stats_registry: - raise ValueError( - f"Timing stats already exist for request_id: {request_id}" - ) - stats = MultiModalProcessorTimingStats() - self.timing_stats_registry[request_id] = stats - return stats - - def clear_timing_stats_registry(self) -> int: - """ - Clear all stats from the registry. Returns the number of stats cleared. - """ - if ( - self.observability_config is None - or not self.observability_config.enable_mm_processor_stats - ): - return 0 - with self._timing_stats_registry_lock: - count = len(self.timing_stats_registry) - self.timing_stats_registry.clear() - return count - - def get_all_timing_stats(self) -> dict[str, dict[str, float]]: - """ - Get all timing stats as a dictionary for API endpoints. - """ - if ( - self.observability_config is None - or not self.observability_config.enable_mm_processor_stats - ): - return {} - with self._timing_stats_registry_lock: - return { - rid: stats.to_dict() - for rid, stats in self.timing_stats_registry.items() - } - - -class BaseProcessingInfo: - """Base class to provide the information necessary for data processing.""" - - def __init__(self, ctx: InputProcessingContext) -> None: - super().__init__() - - self.ctx = ctx - - @property - def model_id(self) -> str: - return self.ctx.model_config.model - - def get_tokenizer(self) -> TokenizerLike: - return self.ctx.get_tokenizer() - - def get_hf_config(self) -> PretrainedConfig: - return self.ctx.get_hf_config() - - def get_hf_processor(self, **kwargs: object) -> ProcessorMixin: - """ - Subclasses can override this method to handle - specific kwargs from model config or user inputs. - """ - return self.ctx.get_hf_processor(**kwargs) - - @property - def skip_prompt_length_check(self) -> bool: - return False - - @abstractmethod - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - """ - Return the maximum supported number of items for each modality. - - A value of `None` means unlimited number of items. - - Omitting a modality from the returned dictionary means that - it is not supported at all. - """ - raise NotImplementedError - - def get_allowed_mm_limits(self) -> Mapping[str, int]: - """Return the maximum allowed number of items for each modality.""" - supported_mm_limits = self.get_supported_mm_limits() - mm_config = self.ctx.get_mm_config() - - allowed_limits = dict[str, int]() - for modality, supported_limit in supported_mm_limits.items(): - user_limit = mm_config.get_limit_per_prompt(modality) - - allowed_limits[modality] = ( - user_limit - if supported_limit is None - else min(user_limit, supported_limit) - ) - - return allowed_limits - - def get_mm_max_tokens_per_item( - self, - seq_len: int, - mm_counts: Mapping[str, int], - ) -> Mapping[str, int] | None: - """ - Return the maximum number of tokens per item of for each modality. - - When `None` (the default) is returned, vLLM will generate dummy inputs - (images/videos) at maximum possible sizes and process them to determine - the maximum token count per modality. - - This approach works but can be very slow for certain models (e.g., - Qwen2.5-VL), leading to very long startup time. For better performance, - each model can override this method to return pre-computed maximum token - counts, avoiding the need for dummy input generation and processing. - - Note: - The maximum number of tokens per item of each modality returned - from this function should respect the model's maximum sequence - length and the maximum number of items of each modality allowed, - and agree with dummy inputs (images/videos) at maximum possible - sizes. - """ - return None - - -_I = TypeVar("_I", bound=BaseProcessingInfo) - -MultiModalHashes = dict[str, list[str]] -""" -A collection of the multi-modal hash for each item, with a similar structure as -[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems]. -""" - MultiModalIsCached = dict[str, list[bool]] """ A collection of the `is_cached` flag for each item, with a similar structure as @@ -1499,6 +957,8 @@ For an item `MultiModalPromptUpdates[k][i]`, `ResolvedPromptUpdate` instances have been applied. """ +_I = TypeVar("_I", bound=BaseProcessingInfo) + class MultiModalProcessingInfo(NamedTuple): kwargs: MultiModalKwargsOptionalItems @@ -1732,7 +1192,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): Call the HF processor on the prompt text and associated multi-modal data. """ - with _timed_operation(self.info.ctx, "hf_processor"): + with timed_operation(self.info.ctx, "hf_processor"): return self.info.ctx.call_hf_processor( self.info.get_hf_processor(**mm_kwargs), dict(text=prompt, **mm_data), @@ -1841,7 +1301,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): Since HF processor requires that text and multi-modal items correspond to each other, we generate dummy text using - [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder] + [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder] to go along with the multi-modal data. """ mm_counts = mm_items.get_all_counts() @@ -2085,7 +1545,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): ) # Use overrides if provided; fallback to data-dependent hashing. - with _timed_operation(self.info.ctx, "hashing"): + with timed_operation(self.info.ctx, "hashing"): mm_hashes = self._hash_mm_items( mm_data_items, hf_processor_mm_kwargs, @@ -2132,7 +1592,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_uuids=mm_uuids, ) - with _timed_operation(self.info.ctx, "hashing"): + with timed_operation(self.info.ctx, "hashing"): mm_hashes = self._hash_mm_items( mm_data_items, hf_processor_mm_kwargs, @@ -2140,7 +1600,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_uuids=mm_uuids, ) - with _timed_operation(self.info.ctx, "cache_lookup"): + with timed_operation(self.info.ctx, "cache_lookup"): mm_is_cached, mm_missing_data_items = self._get_cache_missing_items( cache=cache, mm_data_items=mm_data_items, @@ -2175,7 +1635,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_missing_kwargs, ) - with _timed_operation(self.info.ctx, "cache_lookup"): + with timed_operation(self.info.ctx, "cache_lookup"): mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs( cache, mm_hashes=mm_hashes, @@ -2386,7 +1846,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): ) # NOTE: tokenization_kwargs are not required to init processor - with _timed_operation(self.info.ctx, "prompt_update"): + with timed_operation(self.info.ctx, "prompt_update"): prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates( mm_items=mm_items, prompt_ids=prompt_ids, diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py index f696de6d36..5b1890de0a 100644 --- a/vllm/multimodal/registry.py +++ b/vllm/multimodal/registry.py @@ -12,11 +12,11 @@ from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config from .cache import BaseMultiModalProcessorCache from .inputs import MultiModalInputs from .processing import ( + BaseDummyInputsBuilder, BaseMultiModalProcessor, BaseProcessingInfo, InputProcessingContext, ) -from .profiling import BaseDummyInputsBuilder if TYPE_CHECKING: from vllm.config import ModelConfig, ObservabilityConfig @@ -45,7 +45,7 @@ class ProcessingInfoFactory(Protocol[_I_co]): class DummyInputsBuilderFactory(Protocol[_I]): # type: ignore[misc] """ Constructs a - [`BaseDummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder] + [`BaseDummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder] instance from the context. """ diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 5beff9f98a..a6ab9164cd 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -17,7 +17,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.multimodal.cache import processor_cache_from_config from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict from vllm.multimodal.parse import MultiModalDataParser -from vllm.multimodal.processing import set_request_id +from vllm.multimodal.processing.context import set_request_id from vllm.multimodal.utils import argsort_mm_positions from vllm.pooling_params import PoolingParams from vllm.sampling_params import _SAMPLING_EPS, SamplingParams