[LoRA][Refactor] Unify multimodal LoRA token count hooks (#55071)

Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
linitra24
2026-09-14 13:19:15 +00:00
committed by GitHub
parent 6623fe5b4e
commit 9d4d9aa5bc
23 changed files with 276 additions and 295 deletions
@@ -14,7 +14,7 @@ from ...utils import build_model_context
class _StubModel:
"""Carries only the two attributes the token helpers read from
"""Carries only the attributes the token-count method reads from
`self`, so the real methods can be exercised without constructing
the full `nn.Module` (vision tower, language model, etc.)."""
@@ -22,18 +22,15 @@ class _StubModel:
pooled_grid_length: int
get_num_mm_encoder_tokens = (
LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens
)
get_num_mm_connector_tokens = (
LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens
get_mm_lora_token_counts = (
LlavaNextVideoForConditionalGeneration.get_mm_lora_token_counts
)
@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
def test_num_mm_tokens_match_real_config(model_id):
"""The stored grid lengths must match what `__init__` derives from
the real HF config, and the two helpers must invert each other's
the real HF config, and the LoRA token counts must preserve the
frame-level scaling exactly."""
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
config = ctx.model_config.hf_config
@@ -49,10 +46,13 @@ def test_num_mm_tokens_match_real_config(model_id):
for num_frames in (1, 2, 8, 16, 32):
num_video_tokens = num_frames * pooled_grid_length**2
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=num_video_tokens,
)
assert encoder_tokens == num_frames * patch_grid_length**2
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
assert connector_tokens == num_video_tokens
@@ -72,10 +72,13 @@ def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_fram
num_video_tokens = num_frames * pooled_grid_length**2
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=num_video_tokens,
)
assert encoder_tokens == num_frames * patch_grid_length**2
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
assert connector_tokens == num_video_tokens
@@ -84,5 +87,9 @@ def test_num_mm_tokens_zero():
stub.patch_grid_length = 24
stub.pooled_grid_length = 12
assert get_num_mm_encoder_tokens(stub, 0) == 0
assert get_num_mm_connector_tokens(stub, 0) == 0
assert get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=0,
) == (0, 0)
+15 -20
View File
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems
@@ -687,28 +688,22 @@ class Blip2ForConditionalGeneration(
tower_model="vision_model",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
if num_image_tokens <= 0:
return 0
assert num_image_tokens % self.config.num_query_tokens == 0, (
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
if num_mm_embeds <= 0:
return 0, 0
assert num_mm_embeds % self.config.num_query_tokens == 0, (
"The number of image tokens must be a multiple of "
"the number of query tokens."
)
num_images = num_image_tokens / self.config.num_query_tokens
return num_images * self._vision_tokens_per_image
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
if num_vision_tokens <= 0:
return 0
assert num_vision_tokens % self._vision_tokens_per_image == 0, (
"The number of vision tokens must be a multiple of "
"the number of tokens per image."
num_images = num_mm_embeds // self.config.num_query_tokens
return (
num_images * self._vision_tokens_per_image,
num_images * self.config.num_query_tokens,
)
num_images = num_vision_tokens / self._vision_tokens_per_image
return num_images * self.config.num_query_tokens
+10 -6
View File
@@ -54,6 +54,7 @@ from vllm.model_executor.models.utils import (
)
from vllm.model_executor.models.vision import get_vit_attn_backend
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItem
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.dotsocr import DotsOCRConfig, DotsVisionConfig
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -718,13 +719,16 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
return image_embeds.split(sizes)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
merge_size = self.vision_tower.spatial_merge_size
return num_image_tokens * (merge_size**2)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
merge_size = self.vision_tower.spatial_merge_size
return num_vision_tokens // (merge_size**2)
return num_mm_embeds * (merge_size**2), num_mm_embeds
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
+11 -37
View File
@@ -19,6 +19,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
@@ -686,43 +687,16 @@ class Gemma3ForConditionalGeneration(
tower_model="vision_tower",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
"""
Calculate the number of tokens output by the vision encoder.
The vision encoder processes images into patch embeddings. For Gemma3,
the relationship between prompt placeholder tokens and actual vision
encoder output tokens depends on the patch grid size.
Args:
num_image_tokens: Number of image placeholder tokens in the prompt
(typically mm_tokens_per_image per image)
Returns:
Number of tokens output by the vision encoder
"""
# For Gemma3, the vision encoder outputs tokens_per_side x tokens_per_side
# tokens per image. Since num_image_tokens represents the number of
# connector output tokens (mm_tokens_per_image = 256), and tokens_per_side
# is sqrt(256) = 16, we need to account for the token expansion.
# Based on empirical testing, the multiplier of 16 works correctly.
return num_image_tokens * 16
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
"""
Calculate the number of tokens output by the multimodal connector.
The connector applies projection and normalization but maintains the
token count for Gemma3.
Args:
num_vision_tokens: Number of tokens from vision encoder
Returns:
Number of tokens after connector processing
"""
# The Gemma3 connector maintains a 1:1 token mapping
return num_vision_tokens
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
tower_tokens = num_mm_embeds * 16
return tower_tokens, tower_tokens
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import (
+9 -11
View File
@@ -83,6 +83,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -2378,19 +2379,16 @@ class Glm4vForConditionalGeneration(
tower_model="visual.",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
merge_size = self.config.vision_config.spatial_merge_size
return num_image_tokens * (merge_size**2)
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
merge_size = self.config.vision_config.spatial_merge_size
return num_vision_tokens // (merge_size**2)
return num_mm_embeds * (merge_size**2), num_mm_embeds
@MULTIMODAL_REGISTRY.register_processor(
+14 -13
View File
@@ -14,7 +14,7 @@ from transformers import PretrainedConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import BatchedTensorInputs
from vllm.multimodal.inputs import BatchedTensorInputs, MultiModalKwargsItem
from vllm.multimodal.parse import (
ImageEmbeddingItems,
ImageProcessorItems,
@@ -178,16 +178,17 @@ class H2OVLChatModel(InternVLChatModel):
prefix=prefix,
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
if num_image_tokens <= 0 or self.num_image_token <= 0:
return 0
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
if num_mm_embeds <= 0 or self.num_image_token <= 0:
return 0, 0
num_patches = num_image_tokens // self.num_image_token
return num_patches * (self.patch_tokens + 1)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
if num_vision_tokens <= 0 or self.num_image_token <= 0:
return 0
num_patches = num_vision_tokens // (self.patch_tokens + 1)
return num_patches * self.num_image_token
num_patches = num_mm_embeds // self.num_image_token
tower_tokens = num_patches * (self.patch_tokens + 1)
return tower_tokens, num_patches * self.num_image_token
+9 -14
View File
@@ -39,6 +39,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
@@ -837,20 +838,14 @@ class Idefics3ForConditionalGeneration(
tower_model="model.vision_model",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
hf_config = self.config
scale_factor = hf_config.scale_factor
return num_image_tokens * scale_factor**2
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
hf_config = self.config
scale_factor = hf_config.scale_factor
return num_vision_tokens // scale_factor**2
return num_mm_embeds * scale_factor**2, num_mm_embeds
+10 -17
View File
@@ -31,7 +31,7 @@ import torch
import torch.nn as nn
from torch import Tensor
from transformers.models.whisper.tokenization_whisper import LANGUAGES
from typing_extensions import Self, TypeIs
from typing_extensions import Self, TypeIs, deprecated
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -420,23 +420,16 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
yield
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
"""
Implement this function to enable LoRA support
for the tower module of the multi-modal model.
Given the number of image tokens, output the number of
multi-modal encoder tokens.
"""
...
@deprecated(
"get_num_mm_encoder_tokens is deprecated; use get_mm_lora_token_counts instead."
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: ...
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
"""
Implement this function to enable LoRA support
for the connector module of the multi-modal model.
Given the number of vision tokens, output the number of
multi-modal connector tokens.
"""
...
@deprecated(
"get_num_mm_connector_tokens is deprecated; use "
"get_mm_lora_token_counts instead."
)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: ...
def get_mm_lora_token_counts(
self,
+14 -12
View File
@@ -29,6 +29,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
BatchedTensorInputs,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
@@ -914,19 +915,20 @@ class InternVLChatModel(
tower_model="vision_model",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
if num_image_tokens <= 0 or self.num_image_token <= 0:
return 0
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
if num_mm_embeds <= 0 or self.num_image_token <= 0:
return 0, 0
num_patches = num_image_tokens // self.num_image_token
return num_patches * (self.patch_tokens + 1)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
if num_vision_tokens <= 0 or self.num_image_token <= 0:
return 0
num_patches = num_vision_tokens // (self.patch_tokens + 1)
return num_patches * self.num_image_token
num_patches = num_mm_embeds // self.num_image_token
tower_tokens = num_patches * (self.patch_tokens + 1)
return tower_tokens, num_patches * self.num_image_token
# -- SupportsEncoderCudaGraph protocol methods --
+10 -6
View File
@@ -40,6 +40,7 @@ from vllm.multimodal.inputs import (
ImageItem,
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -1495,13 +1496,16 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
tower_model="visual.",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
merge_size = self.config.vision_config.spatial_merge_size
return num_image_tokens * merge_size**2
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
merge_size = self.config.vision_config.spatial_merge_size
return num_vision_tokens // merge_size**2
return num_mm_embeds * merge_size**2, num_mm_embeds
@MULTIMODAL_REGISTRY.register_processor(
+10 -8
View File
@@ -35,6 +35,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
@@ -1284,12 +1285,13 @@ class Lfm2VLForConditionalGeneration(
tower_model="vision_tower",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
downsample_factor = self.config.downsample_factor
return num_image_tokens * downsample_factor**2
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
downsample_factor = self.config.downsample_factor
return num_vision_tokens // downsample_factor**2
return num_mm_embeds * downsample_factor**2, num_mm_embeds
+9 -14
View File
@@ -28,6 +28,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import BaseMultiModalProcessorCache
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
@@ -739,18 +740,12 @@ class LlavaForConditionalGeneration(
tower_model="vision_tower",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
# LLaVA's vision encoder outputs one token per patch without
# spatial merging or pixel shuffle
return num_image_tokens
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
# LLaVA's MLP projector outputs the same number of tokens
# as it receives from the vision encoder (1:1 mapping)
return num_vision_tokens
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
return num_mm_embeds, num_mm_embeds
+20 -20
View File
@@ -17,6 +17,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
@@ -494,31 +495,30 @@ class LlavaNextVideoForConditionalGeneration(
tower_model="vision_tower",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_video_tokens: int,
) -> int:
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
# Invert the spatial pooling done by `vision_resampler`: each frame
# contributes `pooled_grid_length ** 2` tokens after the language
# model's placeholder count, but `patch_grid_length ** 2` tokens
# when it leaves the vision encoder.
pooled_tokens_per_frame = self.pooled_grid_length**2
if num_video_tokens <= 0 or pooled_tokens_per_frame <= 0:
return 0
num_frames = num_video_tokens // pooled_tokens_per_frame
return num_frames * (self.patch_grid_length**2)
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
# The projector is length-preserving; it runs after
# `vision_resampler` has already pooled each frame down to
# `pooled_grid_length ** 2` tokens.
patch_tokens_per_frame = self.patch_grid_length**2
if num_vision_tokens <= 0 or patch_tokens_per_frame <= 0:
return 0
if (
num_mm_embeds <= 0
or pooled_tokens_per_frame <= 0
or patch_tokens_per_frame <= 0
):
return 0, 0
num_frames = num_vision_tokens // patch_tokens_per_frame
return num_frames * (self.pooled_grid_length**2)
num_frames = num_mm_embeds // pooled_tokens_per_frame
return (
num_frames * patch_tokens_per_frame,
num_frames * pooled_tokens_per_frame,
)
+13 -15
View File
@@ -60,6 +60,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
@@ -1281,21 +1282,18 @@ class Llama4ForConditionalGeneration(
tower_model="vision_model.",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
vision_config = self.config.vision_config
patches_per_chunk = Mllama4ProcessingInfo.get_patch_per_chunk(vision_config)
if num_image_tokens <= 0 or patches_per_chunk <= 0:
return 0
if num_mm_embeds <= 0 or patches_per_chunk <= 0:
return 0, 0
raw_patches = (vision_config.image_size // vision_config.patch_size) ** 2
num_chunks = num_image_tokens // patches_per_chunk
# Encoder processes raw_patches + 1 (CLS) per chunk
return num_chunks * (raw_patches + 1)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
vision_config = self.config.vision_config
raw_patches = (vision_config.image_size // vision_config.patch_size) ** 2
if num_vision_tokens <= 0:
return 0
num_chunks = num_vision_tokens // (raw_patches + 1)
patches_per_chunk = Mllama4ProcessingInfo.get_patch_per_chunk(vision_config)
return num_chunks * patches_per_chunk
num_chunks = num_mm_embeds // patches_per_chunk
return num_chunks * (raw_patches + 1), num_chunks * patches_per_chunk
+10 -5
View File
@@ -47,6 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
@@ -1104,11 +1105,15 @@ class Moondream3ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def get_language_model(self) -> nn.Module:
return self.text
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
return num_image_tokens
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
return num_vision_tokens
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
return num_mm_embeds, num_mm_embeds
def _split_pixel_values(
self,
+14 -6
View File
@@ -24,7 +24,11 @@ from vllm.model_executor.models.interfaces import (
SupportsPP,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
ImageEmbeddingItems,
ImageProcessorItems,
@@ -487,11 +491,15 @@ class OpenVLAForActionPrediction(nn.Module, SupportsMultiModal, SupportsPP):
tower_model="vision_backbone",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
return num_image_tokens
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
return num_vision_tokens
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
return num_mm_embeds, num_mm_embeds
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
def maybe_rename_vision_weights(
+10 -5
View File
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
@@ -401,8 +402,12 @@ class PaliGemmaForConditionalGeneration(
tower_model="vision_tower",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
return num_image_tokens
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
return num_vision_tokens
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
return num_mm_embeds, num_mm_embeds
+11 -9
View File
@@ -42,6 +42,7 @@ from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsOptionalItems,
NestedTensors,
)
@@ -623,17 +624,18 @@ class PixtralForConditionalGeneration(
tower_model="vision_encoder",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
if getattr(self, "patch_merger", None) is None:
return num_image_tokens
return num_mm_embeds, num_mm_embeds
merge_size = self.vision_args.spatial_merge_size
return num_image_tokens * (merge_size**2)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
if getattr(self, "patch_merger", None) is None:
return num_vision_tokens
merge_size = self.vision_args.spatial_merge_size
return num_vision_tokens // (merge_size**2)
return num_mm_embeds * (merge_size**2), num_mm_embeds
# Vision encoder
+9 -14
View File
@@ -70,6 +70,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems
@@ -2050,21 +2051,15 @@ class Qwen2_5_VLForConditionalGeneration(
tower_model="visual.",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_image_tokens * merge_size**2
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_vision_tokens // merge_size**2
return num_mm_embeds * merge_size**2, num_mm_embeds
+9 -14
View File
@@ -66,6 +66,7 @@ from vllm.multimodal.inputs import (
ImageItem,
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
@@ -1747,21 +1748,15 @@ class Qwen2VLForConditionalGeneration(
tower_model="visual.",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_image_tokens * merge_size**2
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_vision_tokens // merge_size**2
return num_mm_embeds * merge_size**2, num_mm_embeds
+10 -6
View File
@@ -67,6 +67,7 @@ from vllm.multimodal.inputs import (
AudioItem,
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
@@ -625,16 +626,19 @@ class Qwen3ASRForConditionalGeneration(
tower_model=["audio_tower."],
)
def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int:
"""Return the number of tokens processed by the audio tower encoder.
Required for LoRA support on the tower module.
"""
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
# For Qwen3-ASR, the audio tower produces one embedding per audio
# placeholder token inserted into the prompt (no additional
# merge/downsample step like vision towers). Therefore, the encoder
# token budget is identity.
return num_audio_tokens
return num_mm_embeds, None
@classmethod
def get_speech_to_text_config(
+8 -14
View File
@@ -3077,24 +3077,18 @@ class Qwen3VLForConditionalGeneration(
tower_model="visual.",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_image_tokens * merge_size**2
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_vision_tokens // merge_size**2
return num_mm_embeds * merge_size**2, num_mm_embeds
@lru_cache
+19 -14
View File
@@ -29,6 +29,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
NestedTensors,
)
@@ -373,9 +374,9 @@ def _build_chunk_attn_metadata(
valid/padding boundary (equivalent to the key-padding mask the HF
implementation uses), while every row still flows through the
(potentially LoRA-wrapped) linears, keeping the per-chunk token counts
constant as required by `get_num_mm_encoder_tokens` /
`get_num_mm_connector_tokens`. Queries at padding positions produce
(garbage) outputs, which are trimmed by `audio_token_len` downstream.
constant as required by `get_mm_lora_token_counts`. Queries at padding
positions produce (garbage) outputs, which are trimmed by `audio_token_len`
downstream.
"""
batch_size = feature_lens.shape[0]
starts = np.arange(batch_size, dtype=np.int64) * seq_len
@@ -684,7 +685,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
# LoRA on the tower/connector requires per-item token counts that are
# predictable from the placeholder count alone (see
# `get_num_mm_encoder_tokens`), so pad every audio chunk's mel input
# `get_mm_lora_token_counts`), so pad every audio chunk's mel input
# to the tower's full context instead of the batch's max length.
self.pad_audio_to_max_context = bool(
lora_config is not None and lora_config.enable_tower_connector_lora
@@ -769,22 +770,26 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
self.config.audio_config.max_source_positions / self.config.stack_factor
)
def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int:
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
# With `pad_audio_to_max_context` (enforced when tower/connector LoRA
# is enabled), the tower's LoRA-wrapped linears always run on
# `max_source_positions` conv-downsampled tokens per chunk, regardless
# of the valid frame count.
num_chunks = math.ceil(num_audio_tokens / self._get_max_tokens_per_chunk())
return num_chunks * self.config.audio_config.max_source_positions
def get_num_mm_connector_tokens(self, num_encoder_tokens: int) -> int:
num_chunks = math.ceil(num_mm_embeds / self._get_max_tokens_per_chunk())
tower_tokens = num_chunks * self.config.audio_config.max_source_positions
# The connector runs on the frame-stacked tower output. Stacking pads
# each chunk to a multiple of `stack_factor`, so this is
# ceil(max_source_positions / stack_factor) tokens per chunk (188 for
# whisper's 1500), not `num_encoder_tokens // stack_factor` (187).
max_source_positions = self.config.audio_config.max_source_positions
num_chunks = num_encoder_tokens // max_source_positions
return num_chunks * self._get_max_tokens_per_chunk()
# whisper's 1500), not `tower_tokens // stack_factor` (187).
connector_tokens = num_chunks * self._get_max_tokens_per_chunk()
return tower_tokens, connector_tokens
def _audio_features_to_embeddings(
self,
@@ -866,7 +871,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
# Pad every chunk to the tower's full context so the number of
# tokens processed by the tower/connector per chunk is constant,
# keeping the LoRA token mappings computed by
# `get_num_mm_encoder_tokens`/`get_num_mm_connector_tokens` exact.
# `get_mm_lora_token_counts` exact.
# Attention to the extra padding is masked via `audio_lens`.
# Over-long inputs are not trimmed here; the tower raises on them.
pad_len = self.audio_tower.max_context_length - audio_features.shape[-1]