diff --git a/tests/models/multimodal/processing/test_llava_next_video.py b/tests/models/multimodal/processing/test_llava_next_video.py index 2dcbb7d4b5..c4fdecd21a 100644 --- a/tests/models/multimodal/processing/test_llava_next_video.py +++ b/tests/models/multimodal/processing/test_llava_next_video.py @@ -14,7 +14,7 @@ from ...utils import build_model_context class _StubModel: - """Carries only the two attributes the token helpers read from + """Carries only the attributes the token-count method reads from `self`, so the real methods can be exercised without constructing the full `nn.Module` (vision tower, language model, etc.).""" @@ -22,18 +22,15 @@ class _StubModel: pooled_grid_length: int -get_num_mm_encoder_tokens = ( - LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens -) -get_num_mm_connector_tokens = ( - LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens +get_mm_lora_token_counts = ( + LlavaNextVideoForConditionalGeneration.get_mm_lora_token_counts ) @pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"]) def test_num_mm_tokens_match_real_config(model_id): """The stored grid lengths must match what `__init__` derives from - the real HF config, and the two helpers must invert each other's + the real HF config, and the LoRA token counts must preserve the frame-level scaling exactly.""" ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1}) config = ctx.model_config.hf_config @@ -49,10 +46,13 @@ def test_num_mm_tokens_match_real_config(model_id): for num_frames in (1, 2, 8, 16, 32): num_video_tokens = num_frames * pooled_grid_length**2 - encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens) + encoder_tokens, connector_tokens = get_mm_lora_token_counts( + stub, + modality="video", + mm_kwargs=None, + num_mm_embeds=num_video_tokens, + ) assert encoder_tokens == num_frames * patch_grid_length**2 - - connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens) assert connector_tokens == num_video_tokens @@ -72,10 +72,13 @@ def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_fram num_video_tokens = num_frames * pooled_grid_length**2 - encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens) + encoder_tokens, connector_tokens = get_mm_lora_token_counts( + stub, + modality="video", + mm_kwargs=None, + num_mm_embeds=num_video_tokens, + ) assert encoder_tokens == num_frames * patch_grid_length**2 - - connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens) assert connector_tokens == num_video_tokens @@ -84,5 +87,9 @@ def test_num_mm_tokens_zero(): stub.patch_grid_length = 24 stub.pooled_grid_length = 12 - assert get_num_mm_encoder_tokens(stub, 0) == 0 - assert get_num_mm_connector_tokens(stub, 0) == 0 + assert get_mm_lora_token_counts( + stub, + modality="video", + mm_kwargs=None, + num_mm_embeds=0, + ) == (0, 0) diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index 091001fc01..de74d1f5ac 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import MultiModalDataItems @@ -687,28 +688,22 @@ class Blip2ForConditionalGeneration( tower_model="vision_model", ) - def get_num_mm_encoder_tokens( + def get_mm_lora_token_counts( self, - num_image_tokens: int, - ) -> int: - if num_image_tokens <= 0: - return 0 - assert num_image_tokens % self.config.num_query_tokens == 0, ( + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + if num_mm_embeds <= 0: + return 0, 0 + assert num_mm_embeds % self.config.num_query_tokens == 0, ( "The number of image tokens must be a multiple of " "the number of query tokens." ) - num_images = num_image_tokens / self.config.num_query_tokens - return num_images * self._vision_tokens_per_image - - def get_num_mm_connector_tokens( - self, - num_vision_tokens: int, - ) -> int: - if num_vision_tokens <= 0: - return 0 - assert num_vision_tokens % self._vision_tokens_per_image == 0, ( - "The number of vision tokens must be a multiple of " - "the number of tokens per image." + num_images = num_mm_embeds // self.config.num_query_tokens + return ( + num_images * self._vision_tokens_per_image, + num_images * self.config.num_query_tokens, ) - num_images = num_vision_tokens / self._vision_tokens_per_image - return num_images * self.config.num_query_tokens diff --git a/vllm/model_executor/models/dots_ocr.py b/vllm/model_executor/models/dots_ocr.py index e91d6ce94c..c939c2bef5 100644 --- a/vllm/model_executor/models/dots_ocr.py +++ b/vllm/model_executor/models/dots_ocr.py @@ -54,6 +54,7 @@ from vllm.model_executor.models.utils import ( ) from vllm.model_executor.models.vision import get_vit_attn_backend from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalKwargsItem from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.dotsocr import DotsOCRConfig, DotsVisionConfig from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -718,13 +719,16 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA return image_embeds.split(sizes) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs merge_size = self.vision_tower.spatial_merge_size - return num_image_tokens * (merge_size**2) - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - merge_size = self.vision_tower.spatial_merge_size - return num_vision_tokens // (merge_size**2) + return num_mm_embeds * (merge_size**2), num_mm_embeds def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: image_input = self._parse_and_validate_image_input(**kwargs) diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index b45327fbeb..abe1f5f1bd 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -19,6 +19,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems @@ -686,43 +687,16 @@ class Gemma3ForConditionalGeneration( tower_model="vision_tower", ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: - """ - Calculate the number of tokens output by the vision encoder. - - The vision encoder processes images into patch embeddings. For Gemma3, - the relationship between prompt placeholder tokens and actual vision - encoder output tokens depends on the patch grid size. - - Args: - num_image_tokens: Number of image placeholder tokens in the prompt - (typically mm_tokens_per_image per image) - - Returns: - Number of tokens output by the vision encoder - """ - # For Gemma3, the vision encoder outputs tokens_per_side x tokens_per_side - # tokens per image. Since num_image_tokens represents the number of - # connector output tokens (mm_tokens_per_image = 256), and tokens_per_side - # is sqrt(256) = 16, we need to account for the token expansion. - # Based on empirical testing, the multiplier of 16 works correctly. - return num_image_tokens * 16 - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - """ - Calculate the number of tokens output by the multimodal connector. - - The connector applies projection and normalization but maintains the - token count for Gemma3. - - Args: - num_vision_tokens: Number of tokens from vision encoder - - Returns: - Number of tokens after connector processing - """ - # The Gemma3 connector maintains a 1:1 token mapping - return num_vision_tokens + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + tower_tokens = num_mm_embeds * 16 + return tower_tokens, tower_tokens def get_encoder_cudagraph_config(self): from vllm.v1.worker.encoder_cudagraph_defs import ( diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 163115ff3d..730a015cfe 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -83,6 +83,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFeatureSpec, MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, VideoItem, ) @@ -2378,19 +2379,16 @@ class Glm4vForConditionalGeneration( tower_model="visual.", ) - def get_num_mm_encoder_tokens( + def get_mm_lora_token_counts( self, - num_image_tokens: int, - ) -> int: + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs merge_size = self.config.vision_config.spatial_merge_size - return num_image_tokens * (merge_size**2) - - def get_num_mm_connector_tokens( - self, - num_vision_tokens: int, - ) -> int: - merge_size = self.config.vision_config.spatial_merge_size - return num_vision_tokens // (merge_size**2) + return num_mm_embeds * (merge_size**2), num_mm_embeds @MULTIMODAL_REGISTRY.register_processor( diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index 5c4f984c13..5be2d7b46d 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -14,7 +14,7 @@ from transformers import PretrainedConfig from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import BatchedTensorInputs +from vllm.multimodal.inputs import BatchedTensorInputs, MultiModalKwargsItem from vllm.multimodal.parse import ( ImageEmbeddingItems, ImageProcessorItems, @@ -178,16 +178,17 @@ class H2OVLChatModel(InternVLChatModel): prefix=prefix, ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: - if num_image_tokens <= 0 or self.num_image_token <= 0: - return 0 + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + if num_mm_embeds <= 0 or self.num_image_token <= 0: + return 0, 0 - num_patches = num_image_tokens // self.num_image_token - return num_patches * (self.patch_tokens + 1) - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - if num_vision_tokens <= 0 or self.num_image_token <= 0: - return 0 - - num_patches = num_vision_tokens // (self.patch_tokens + 1) - return num_patches * self.num_image_token + num_patches = num_mm_embeds // self.num_image_token + tower_tokens = num_patches * (self.patch_tokens + 1) + return tower_tokens, num_patches * self.num_image_token diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index baaa4106cb..4ac8eb5eeb 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -39,6 +39,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems @@ -837,20 +838,14 @@ class Idefics3ForConditionalGeneration( tower_model="model.vision_model", ) - def get_num_mm_encoder_tokens( + def get_mm_lora_token_counts( self, - num_image_tokens: int, - ) -> int: + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs hf_config = self.config scale_factor = hf_config.scale_factor - - return num_image_tokens * scale_factor**2 - - def get_num_mm_connector_tokens( - self, - num_vision_tokens: int, - ) -> int: - hf_config = self.config - scale_factor = hf_config.scale_factor - - return num_vision_tokens // scale_factor**2 + return num_mm_embeds * scale_factor**2, num_mm_embeds diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 254f2c635b..2e79001add 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -31,7 +31,7 @@ import torch import torch.nn as nn from torch import Tensor from transformers.models.whisper.tokenization_whisper import LANGUAGES -from typing_extensions import Self, TypeIs +from typing_extensions import Self, TypeIs, deprecated from vllm.logger import init_logger from vllm.model_executor.layers.quantization import QuantizationConfig @@ -420,23 +420,16 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol): yield - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: - """ - Implement this function to enable LoRA support - for the tower module of the multi-modal model. - Given the number of image tokens, output the number of - multi-modal encoder tokens. - """ - ... + @deprecated( + "get_num_mm_encoder_tokens is deprecated; use get_mm_lora_token_counts instead." + ) + def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: ... - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - """ - Implement this function to enable LoRA support - for the connector module of the multi-modal model. - Given the number of vision tokens, output the number of - multi-modal connector tokens. - """ - ... + @deprecated( + "get_num_mm_connector_tokens is deprecated; use " + "get_mm_lora_token_counts instead." + ) + def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: ... def get_mm_lora_token_counts( self, diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index 1fcb6b8681..c57359b7b5 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -29,6 +29,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( BatchedTensorInputs, MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ( @@ -914,19 +915,20 @@ class InternVLChatModel( tower_model="vision_model", ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: - if num_image_tokens <= 0 or self.num_image_token <= 0: - return 0 + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + if num_mm_embeds <= 0 or self.num_image_token <= 0: + return 0, 0 - num_patches = num_image_tokens // self.num_image_token - return num_patches * (self.patch_tokens + 1) - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - if num_vision_tokens <= 0 or self.num_image_token <= 0: - return 0 - - num_patches = num_vision_tokens // (self.patch_tokens + 1) - return num_patches * self.num_image_token + num_patches = num_mm_embeds // self.num_image_token + tower_tokens = num_patches * (self.patch_tokens + 1) + return tower_tokens, num_patches * self.num_image_token # -- SupportsEncoderCudaGraph protocol methods -- diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 53cda0ed17..d117b91d53 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -40,6 +40,7 @@ from vllm.multimodal.inputs import ( ImageItem, MultiModalFeatureSpec, MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, VideoItem, ) @@ -1495,13 +1496,16 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): tower_model="visual.", ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs merge_size = self.config.vision_config.spatial_merge_size - return num_image_tokens * merge_size**2 - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - merge_size = self.config.vision_config.spatial_merge_size - return num_vision_tokens // merge_size**2 + return num_mm_embeds * merge_size**2, num_mm_embeds @MULTIMODAL_REGISTRY.register_processor( diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index a8b57e7c28..15872a1f3f 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -35,6 +35,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems @@ -1284,12 +1285,13 @@ class Lfm2VLForConditionalGeneration( tower_model="vision_tower", ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs downsample_factor = self.config.downsample_factor - - return num_image_tokens * downsample_factor**2 - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - downsample_factor = self.config.downsample_factor - - return num_vision_tokens // downsample_factor**2 + return num_mm_embeds * downsample_factor**2, num_mm_embeds diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index 1f1f45bfea..a32e0e6c63 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -28,6 +28,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import BaseMultiModalProcessorCache from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ( @@ -739,18 +740,12 @@ class LlavaForConditionalGeneration( tower_model="vision_tower", ) - def get_num_mm_encoder_tokens( + def get_mm_lora_token_counts( self, - num_image_tokens: int, - ) -> int: - # LLaVA's vision encoder outputs one token per patch without - # spatial merging or pixel shuffle - return num_image_tokens - - def get_num_mm_connector_tokens( - self, - num_vision_tokens: int, - ) -> int: - # LLaVA's MLP projector outputs the same number of tokens - # as it receives from the vision encoder (1:1 mapping) - return num_vision_tokens + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + return num_mm_embeds, num_mm_embeds diff --git a/vllm/model_executor/models/llava_next_video.py b/vllm/model_executor/models/llava_next_video.py index 038da17b75..96ebdaea8e 100644 --- a/vllm/model_executor/models/llava_next_video.py +++ b/vllm/model_executor/models/llava_next_video.py @@ -17,6 +17,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ( @@ -494,31 +495,30 @@ class LlavaNextVideoForConditionalGeneration( tower_model="vision_tower", ) - def get_num_mm_encoder_tokens( + def get_mm_lora_token_counts( self, - num_video_tokens: int, - ) -> int: + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + # Invert the spatial pooling done by `vision_resampler`: each frame # contributes `pooled_grid_length ** 2` tokens after the language # model's placeholder count, but `patch_grid_length ** 2` tokens # when it leaves the vision encoder. pooled_tokens_per_frame = self.pooled_grid_length**2 - if num_video_tokens <= 0 or pooled_tokens_per_frame <= 0: - return 0 - - num_frames = num_video_tokens // pooled_tokens_per_frame - return num_frames * (self.patch_grid_length**2) - - def get_num_mm_connector_tokens( - self, - num_vision_tokens: int, - ) -> int: - # The projector is length-preserving; it runs after - # `vision_resampler` has already pooled each frame down to - # `pooled_grid_length ** 2` tokens. patch_tokens_per_frame = self.patch_grid_length**2 - if num_vision_tokens <= 0 or patch_tokens_per_frame <= 0: - return 0 + if ( + num_mm_embeds <= 0 + or pooled_tokens_per_frame <= 0 + or patch_tokens_per_frame <= 0 + ): + return 0, 0 - num_frames = num_vision_tokens // patch_tokens_per_frame - return num_frames * (self.pooled_grid_length**2) + num_frames = num_mm_embeds // pooled_tokens_per_frame + return ( + num_frames * patch_tokens_per_frame, + num_frames * pooled_tokens_per_frame, + ) diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 5a6e654417..0096eb4241 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -60,6 +60,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems @@ -1281,21 +1282,18 @@ class Llama4ForConditionalGeneration( tower_model="vision_model.", ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs vision_config = self.config.vision_config patches_per_chunk = Mllama4ProcessingInfo.get_patch_per_chunk(vision_config) - if num_image_tokens <= 0 or patches_per_chunk <= 0: - return 0 + if num_mm_embeds <= 0 or patches_per_chunk <= 0: + return 0, 0 raw_patches = (vision_config.image_size // vision_config.patch_size) ** 2 - num_chunks = num_image_tokens // patches_per_chunk - # Encoder processes raw_patches + 1 (CLS) per chunk - return num_chunks * (raw_patches + 1) - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - vision_config = self.config.vision_config - raw_patches = (vision_config.image_size // vision_config.patch_size) ** 2 - if num_vision_tokens <= 0: - return 0 - num_chunks = num_vision_tokens // (raw_patches + 1) - patches_per_chunk = Mllama4ProcessingInfo.get_patch_per_chunk(vision_config) - return num_chunks * patches_per_chunk + num_chunks = num_mm_embeds // patches_per_chunk + return num_chunks * (raw_patches + 1), num_chunks * patches_per_chunk diff --git a/vllm/model_executor/models/moondream3.py b/vllm/model_executor/models/moondream3.py index 818015e1ba..47a240b589 100644 --- a/vllm/model_executor/models/moondream3.py +++ b/vllm/model_executor/models/moondream3.py @@ -47,6 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ImageSize, MultiModalDataItems @@ -1104,11 +1105,15 @@ class Moondream3ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP): def get_language_model(self) -> nn.Module: return self.text - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: - return num_image_tokens - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - return num_vision_tokens + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + return num_mm_embeds, num_mm_embeds def _split_pixel_values( self, diff --git a/vllm/model_executor/models/openvla.py b/vllm/model_executor/models/openvla.py index a676b3447c..ecbfb8a54c 100644 --- a/vllm/model_executor/models/openvla.py +++ b/vllm/model_executor/models/openvla.py @@ -24,7 +24,11 @@ from vllm.model_executor.models.interfaces import ( SupportsPP, ) from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems +from vllm.multimodal.inputs import ( + MultiModalFieldConfig, + MultiModalKwargsItem, + MultiModalKwargsItems, +) from vllm.multimodal.parse import ( ImageEmbeddingItems, ImageProcessorItems, @@ -487,11 +491,15 @@ class OpenVLAForActionPrediction(nn.Module, SupportsMultiModal, SupportsPP): tower_model="vision_backbone", ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: - return num_image_tokens - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - return num_vision_tokens + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + return num_mm_embeds, num_mm_embeds def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def maybe_rename_vision_weights( diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py index 2fa498ee26..8e3524eeec 100644 --- a/vllm/model_executor/models/paligemma.py +++ b/vllm/model_executor/models/paligemma.py @@ -14,6 +14,7 @@ from vllm.logger import init_logger from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ( @@ -401,8 +402,12 @@ class PaliGemmaForConditionalGeneration( tower_model="vision_tower", ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: - return num_image_tokens - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - return num_vision_tokens + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs + return num_mm_embeds, num_mm_embeds diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index b6848b09de..1485b90e39 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -42,6 +42,7 @@ from vllm.model_executor.models.utils import WeightsMapper from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsOptionalItems, NestedTensors, ) @@ -623,17 +624,18 @@ class PixtralForConditionalGeneration( tower_model="vision_encoder", ) - def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs if getattr(self, "patch_merger", None) is None: - return num_image_tokens + return num_mm_embeds, num_mm_embeds merge_size = self.vision_args.spatial_merge_size - return num_image_tokens * (merge_size**2) - - def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: - if getattr(self, "patch_merger", None) is None: - return num_vision_tokens - merge_size = self.vision_args.spatial_merge_size - return num_vision_tokens // (merge_size**2) + return num_mm_embeds * (merge_size**2), num_mm_embeds # Vision encoder diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 61d0153f86..6cfacc1421 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -70,6 +70,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFeatureSpec, MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import MultiModalDataItems @@ -2050,21 +2051,15 @@ class Qwen2_5_VLForConditionalGeneration( tower_model="visual.", ) - def get_num_mm_encoder_tokens( + def get_mm_lora_token_counts( self, - num_image_tokens: int, - ) -> int: + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs hf_config = self.config vision_config = hf_config.vision_config merge_size = vision_config.spatial_merge_size - - return num_image_tokens * merge_size**2 - - def get_num_mm_connector_tokens( - self, - num_vision_tokens: int, - ) -> int: - hf_config = self.config - vision_config = hf_config.vision_config - merge_size = vision_config.spatial_merge_size - return num_vision_tokens // merge_size**2 + return num_mm_embeds * merge_size**2, num_mm_embeds diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 80abdc62b5..c3761f8893 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -66,6 +66,7 @@ from vllm.multimodal.inputs import ( ImageItem, MultiModalFeatureSpec, MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, VideoItem, ) @@ -1747,21 +1748,15 @@ class Qwen2VLForConditionalGeneration( tower_model="visual.", ) - def get_num_mm_encoder_tokens( + def get_mm_lora_token_counts( self, - num_image_tokens: int, - ) -> int: + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs hf_config = self.config vision_config = hf_config.vision_config merge_size = vision_config.spatial_merge_size - - return num_image_tokens * merge_size**2 - - def get_num_mm_connector_tokens( - self, - num_vision_tokens: int, - ) -> int: - hf_config = self.config - vision_config = hf_config.vision_config - merge_size = vision_config.spatial_merge_size - return num_vision_tokens // merge_size**2 + return num_mm_embeds * merge_size**2, num_mm_embeds diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index e98f37ba37..3bea939340 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -67,6 +67,7 @@ from vllm.multimodal.inputs import ( AudioItem, MultiModalFeatureSpec, MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, ) from vllm.multimodal.parse import ( @@ -625,16 +626,19 @@ class Qwen3ASRForConditionalGeneration( tower_model=["audio_tower."], ) - def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int: - """Return the number of tokens processed by the audio tower encoder. - - Required for LoRA support on the tower module. - """ + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs # For Qwen3-ASR, the audio tower produces one embedding per audio # placeholder token inserted into the prompt (no additional # merge/downsample step like vision towers). Therefore, the encoder # token budget is identity. - return num_audio_tokens + return num_mm_embeds, None @classmethod def get_speech_to_text_config( diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index d99cfe7aa2..644fd134ad 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -3077,24 +3077,18 @@ class Qwen3VLForConditionalGeneration( tower_model="visual.", ) - def get_num_mm_encoder_tokens( + def get_mm_lora_token_counts( self, - num_image_tokens: int, - ) -> int: + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs hf_config = self.config vision_config = hf_config.vision_config merge_size = vision_config.spatial_merge_size - - return num_image_tokens * merge_size**2 - - def get_num_mm_connector_tokens( - self, - num_vision_tokens: int, - ) -> int: - hf_config = self.config - vision_config = hf_config.vision_config - merge_size = vision_config.spatial_merge_size - return num_vision_tokens // merge_size**2 + return num_mm_embeds * merge_size**2, num_mm_embeds @lru_cache diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index a394e5f02e..bffe94fdce 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -29,6 +29,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsItem, MultiModalKwargsItems, NestedTensors, ) @@ -373,9 +374,9 @@ def _build_chunk_attn_metadata( valid/padding boundary (equivalent to the key-padding mask the HF implementation uses), while every row still flows through the (potentially LoRA-wrapped) linears, keeping the per-chunk token counts - constant as required by `get_num_mm_encoder_tokens` / - `get_num_mm_connector_tokens`. Queries at padding positions produce - (garbage) outputs, which are trimmed by `audio_token_len` downstream. + constant as required by `get_mm_lora_token_counts`. Queries at padding + positions produce (garbage) outputs, which are trimmed by `audio_token_len` + downstream. """ batch_size = feature_lens.shape[0] starts = np.arange(batch_size, dtype=np.int64) * seq_len @@ -684,7 +685,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA): # LoRA on the tower/connector requires per-item token counts that are # predictable from the placeholder count alone (see - # `get_num_mm_encoder_tokens`), so pad every audio chunk's mel input + # `get_mm_lora_token_counts`), so pad every audio chunk's mel input # to the tower's full context instead of the batch's max length. self.pad_audio_to_max_context = bool( lora_config is not None and lora_config.enable_tower_connector_lora @@ -769,22 +770,26 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA): self.config.audio_config.max_source_positions / self.config.stack_factor ) - def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int: + def get_mm_lora_token_counts( + self, + *, + modality: str, + mm_kwargs: MultiModalKwargsItem | None, + num_mm_embeds: int, + ) -> tuple[int, int | None]: + del modality, mm_kwargs # With `pad_audio_to_max_context` (enforced when tower/connector LoRA # is enabled), the tower's LoRA-wrapped linears always run on # `max_source_positions` conv-downsampled tokens per chunk, regardless # of the valid frame count. - num_chunks = math.ceil(num_audio_tokens / self._get_max_tokens_per_chunk()) - return num_chunks * self.config.audio_config.max_source_positions - - def get_num_mm_connector_tokens(self, num_encoder_tokens: int) -> int: + num_chunks = math.ceil(num_mm_embeds / self._get_max_tokens_per_chunk()) + tower_tokens = num_chunks * self.config.audio_config.max_source_positions # The connector runs on the frame-stacked tower output. Stacking pads # each chunk to a multiple of `stack_factor`, so this is # ceil(max_source_positions / stack_factor) tokens per chunk (188 for - # whisper's 1500), not `num_encoder_tokens // stack_factor` (187). - max_source_positions = self.config.audio_config.max_source_positions - num_chunks = num_encoder_tokens // max_source_positions - return num_chunks * self._get_max_tokens_per_chunk() + # whisper's 1500), not `tower_tokens // stack_factor` (187). + connector_tokens = num_chunks * self._get_max_tokens_per_chunk() + return tower_tokens, connector_tokens def _audio_features_to_embeddings( self, @@ -866,7 +871,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA): # Pad every chunk to the tower's full context so the number of # tokens processed by the tower/connector per chunk is constant, # keeping the LoRA token mappings computed by - # `get_num_mm_encoder_tokens`/`get_num_mm_connector_tokens` exact. + # `get_mm_lora_token_counts` exact. # Attention to the extra padding is masked via `audio_lens`. # Over-long inputs are not trimmed here; the tower raises on them. pad_len = self.audio_tower.max_context_length - audio_features.shape[-1]