[LoRA][Refactor] Unify multimodal LoRA token count hooks (#55071)
Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
@@ -14,7 +14,7 @@ from ...utils import build_model_context
|
||||
|
||||
|
||||
class _StubModel:
|
||||
"""Carries only the two attributes the token helpers read from
|
||||
"""Carries only the attributes the token-count method reads from
|
||||
`self`, so the real methods can be exercised without constructing
|
||||
the full `nn.Module` (vision tower, language model, etc.)."""
|
||||
|
||||
@@ -22,18 +22,15 @@ class _StubModel:
|
||||
pooled_grid_length: int
|
||||
|
||||
|
||||
get_num_mm_encoder_tokens = (
|
||||
LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens
|
||||
)
|
||||
get_num_mm_connector_tokens = (
|
||||
LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens
|
||||
get_mm_lora_token_counts = (
|
||||
LlavaNextVideoForConditionalGeneration.get_mm_lora_token_counts
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
|
||||
def test_num_mm_tokens_match_real_config(model_id):
|
||||
"""The stored grid lengths must match what `__init__` derives from
|
||||
the real HF config, and the two helpers must invert each other's
|
||||
the real HF config, and the LoRA token counts must preserve the
|
||||
frame-level scaling exactly."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
|
||||
config = ctx.model_config.hf_config
|
||||
@@ -49,10 +46,13 @@ def test_num_mm_tokens_match_real_config(model_id):
|
||||
for num_frames in (1, 2, 8, 16, 32):
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
|
||||
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=num_video_tokens,
|
||||
)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
|
||||
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
@@ -72,10 +72,13 @@ def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_fram
|
||||
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
|
||||
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=num_video_tokens,
|
||||
)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
|
||||
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
@@ -84,5 +87,9 @@ def test_num_mm_tokens_zero():
|
||||
stub.patch_grid_length = 24
|
||||
stub.pooled_grid_length = 12
|
||||
|
||||
assert get_num_mm_encoder_tokens(stub, 0) == 0
|
||||
assert get_num_mm_connector_tokens(stub, 0) == 0
|
||||
assert get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=0,
|
||||
) == (0, 0)
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
@@ -687,28 +688,22 @@ class Blip2ForConditionalGeneration(
|
||||
tower_model="vision_model",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
if num_image_tokens <= 0:
|
||||
return 0
|
||||
assert num_image_tokens % self.config.num_query_tokens == 0, (
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
if num_mm_embeds <= 0:
|
||||
return 0, 0
|
||||
assert num_mm_embeds % self.config.num_query_tokens == 0, (
|
||||
"The number of image tokens must be a multiple of "
|
||||
"the number of query tokens."
|
||||
)
|
||||
num_images = num_image_tokens / self.config.num_query_tokens
|
||||
return num_images * self._vision_tokens_per_image
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
if num_vision_tokens <= 0:
|
||||
return 0
|
||||
assert num_vision_tokens % self._vision_tokens_per_image == 0, (
|
||||
"The number of vision tokens must be a multiple of "
|
||||
"the number of tokens per image."
|
||||
num_images = num_mm_embeds // self.config.num_query_tokens
|
||||
return (
|
||||
num_images * self._vision_tokens_per_image,
|
||||
num_images * self.config.num_query_tokens,
|
||||
)
|
||||
num_images = num_vision_tokens / self._vision_tokens_per_image
|
||||
return num_images * self.config.num_query_tokens
|
||||
|
||||
@@ -54,6 +54,7 @@ from vllm.model_executor.models.utils import (
|
||||
)
|
||||
from vllm.model_executor.models.vision import get_vit_attn_backend
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItem
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.dotsocr import DotsOCRConfig, DotsVisionConfig
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
@@ -718,13 +719,16 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
|
||||
return image_embeds.split(sizes)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
merge_size = self.vision_tower.spatial_merge_size
|
||||
return num_image_tokens * (merge_size**2)
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
merge_size = self.vision_tower.spatial_merge_size
|
||||
return num_vision_tokens // (merge_size**2)
|
||||
return num_mm_embeds * (merge_size**2), num_mm_embeds
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
|
||||
@@ -686,43 +687,16 @@ class Gemma3ForConditionalGeneration(
|
||||
tower_model="vision_tower",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
"""
|
||||
Calculate the number of tokens output by the vision encoder.
|
||||
|
||||
The vision encoder processes images into patch embeddings. For Gemma3,
|
||||
the relationship between prompt placeholder tokens and actual vision
|
||||
encoder output tokens depends on the patch grid size.
|
||||
|
||||
Args:
|
||||
num_image_tokens: Number of image placeholder tokens in the prompt
|
||||
(typically mm_tokens_per_image per image)
|
||||
|
||||
Returns:
|
||||
Number of tokens output by the vision encoder
|
||||
"""
|
||||
# For Gemma3, the vision encoder outputs tokens_per_side x tokens_per_side
|
||||
# tokens per image. Since num_image_tokens represents the number of
|
||||
# connector output tokens (mm_tokens_per_image = 256), and tokens_per_side
|
||||
# is sqrt(256) = 16, we need to account for the token expansion.
|
||||
# Based on empirical testing, the multiplier of 16 works correctly.
|
||||
return num_image_tokens * 16
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
"""
|
||||
Calculate the number of tokens output by the multimodal connector.
|
||||
|
||||
The connector applies projection and normalization but maintains the
|
||||
token count for Gemma3.
|
||||
|
||||
Args:
|
||||
num_vision_tokens: Number of tokens from vision encoder
|
||||
|
||||
Returns:
|
||||
Number of tokens after connector processing
|
||||
"""
|
||||
# The Gemma3 connector maintains a 1:1 token mapping
|
||||
return num_vision_tokens
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
tower_tokens = num_mm_embeds * 16
|
||||
return tower_tokens, tower_tokens
|
||||
|
||||
def get_encoder_cudagraph_config(self):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
|
||||
@@ -83,6 +83,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -2378,19 +2379,16 @@ class Glm4vForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
merge_size = self.config.vision_config.spatial_merge_size
|
||||
return num_image_tokens * (merge_size**2)
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
merge_size = self.config.vision_config.spatial_merge_size
|
||||
return num_vision_tokens // (merge_size**2)
|
||||
return num_mm_embeds * (merge_size**2), num_mm_embeds
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
|
||||
@@ -14,7 +14,7 @@ from transformers import PretrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import BatchedTensorInputs
|
||||
from vllm.multimodal.inputs import BatchedTensorInputs, MultiModalKwargsItem
|
||||
from vllm.multimodal.parse import (
|
||||
ImageEmbeddingItems,
|
||||
ImageProcessorItems,
|
||||
@@ -178,16 +178,17 @@ class H2OVLChatModel(InternVLChatModel):
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
if num_image_tokens <= 0 or self.num_image_token <= 0:
|
||||
return 0
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
if num_mm_embeds <= 0 or self.num_image_token <= 0:
|
||||
return 0, 0
|
||||
|
||||
num_patches = num_image_tokens // self.num_image_token
|
||||
return num_patches * (self.patch_tokens + 1)
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
if num_vision_tokens <= 0 or self.num_image_token <= 0:
|
||||
return 0
|
||||
|
||||
num_patches = num_vision_tokens // (self.patch_tokens + 1)
|
||||
return num_patches * self.num_image_token
|
||||
num_patches = num_mm_embeds // self.num_image_token
|
||||
tower_tokens = num_patches * (self.patch_tokens + 1)
|
||||
return tower_tokens, num_patches * self.num_image_token
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
|
||||
@@ -837,20 +838,14 @@ class Idefics3ForConditionalGeneration(
|
||||
tower_model="model.vision_model",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
scale_factor = hf_config.scale_factor
|
||||
|
||||
return num_image_tokens * scale_factor**2
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
scale_factor = hf_config.scale_factor
|
||||
|
||||
return num_vision_tokens // scale_factor**2
|
||||
return num_mm_embeds * scale_factor**2, num_mm_embeds
|
||||
|
||||
@@ -31,7 +31,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
||||
from typing_extensions import Self, TypeIs
|
||||
from typing_extensions import Self, TypeIs, deprecated
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -420,23 +420,16 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
|
||||
yield
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
"""
|
||||
Implement this function to enable LoRA support
|
||||
for the tower module of the multi-modal model.
|
||||
Given the number of image tokens, output the number of
|
||||
multi-modal encoder tokens.
|
||||
"""
|
||||
...
|
||||
@deprecated(
|
||||
"get_num_mm_encoder_tokens is deprecated; use get_mm_lora_token_counts instead."
|
||||
)
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: ...
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
"""
|
||||
Implement this function to enable LoRA support
|
||||
for the connector module of the multi-modal model.
|
||||
Given the number of vision tokens, output the number of
|
||||
multi-modal connector tokens.
|
||||
"""
|
||||
...
|
||||
@deprecated(
|
||||
"get_num_mm_connector_tokens is deprecated; use "
|
||||
"get_mm_lora_token_counts instead."
|
||||
)
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: ...
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
BatchedTensorInputs,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -914,19 +915,20 @@ class InternVLChatModel(
|
||||
tower_model="vision_model",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
if num_image_tokens <= 0 or self.num_image_token <= 0:
|
||||
return 0
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
if num_mm_embeds <= 0 or self.num_image_token <= 0:
|
||||
return 0, 0
|
||||
|
||||
num_patches = num_image_tokens // self.num_image_token
|
||||
return num_patches * (self.patch_tokens + 1)
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
if num_vision_tokens <= 0 or self.num_image_token <= 0:
|
||||
return 0
|
||||
|
||||
num_patches = num_vision_tokens // (self.patch_tokens + 1)
|
||||
return num_patches * self.num_image_token
|
||||
num_patches = num_mm_embeds // self.num_image_token
|
||||
tower_tokens = num_patches * (self.patch_tokens + 1)
|
||||
return tower_tokens, num_patches * self.num_image_token
|
||||
|
||||
# -- SupportsEncoderCudaGraph protocol methods --
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.multimodal.inputs import (
|
||||
ImageItem,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -1495,13 +1496,16 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
merge_size = self.config.vision_config.spatial_merge_size
|
||||
return num_image_tokens * merge_size**2
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
merge_size = self.config.vision_config.spatial_merge_size
|
||||
return num_vision_tokens // merge_size**2
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
|
||||
@@ -1284,12 +1285,13 @@ class Lfm2VLForConditionalGeneration(
|
||||
tower_model="vision_tower",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
downsample_factor = self.config.downsample_factor
|
||||
|
||||
return num_image_tokens * downsample_factor**2
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
downsample_factor = self.config.downsample_factor
|
||||
|
||||
return num_vision_tokens // downsample_factor**2
|
||||
return num_mm_embeds * downsample_factor**2, num_mm_embeds
|
||||
|
||||
@@ -28,6 +28,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -739,18 +740,12 @@ class LlavaForConditionalGeneration(
|
||||
tower_model="vision_tower",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
# LLaVA's vision encoder outputs one token per patch without
|
||||
# spatial merging or pixel shuffle
|
||||
return num_image_tokens
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
# LLaVA's MLP projector outputs the same number of tokens
|
||||
# as it receives from the vision encoder (1:1 mapping)
|
||||
return num_vision_tokens
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
return num_mm_embeds, num_mm_embeds
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.models.clip import CLIPVisionModel
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -494,31 +495,30 @@ class LlavaNextVideoForConditionalGeneration(
|
||||
tower_model="vision_tower",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_video_tokens: int,
|
||||
) -> int:
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
|
||||
# Invert the spatial pooling done by `vision_resampler`: each frame
|
||||
# contributes `pooled_grid_length ** 2` tokens after the language
|
||||
# model's placeholder count, but `patch_grid_length ** 2` tokens
|
||||
# when it leaves the vision encoder.
|
||||
pooled_tokens_per_frame = self.pooled_grid_length**2
|
||||
if num_video_tokens <= 0 or pooled_tokens_per_frame <= 0:
|
||||
return 0
|
||||
|
||||
num_frames = num_video_tokens // pooled_tokens_per_frame
|
||||
return num_frames * (self.patch_grid_length**2)
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
# The projector is length-preserving; it runs after
|
||||
# `vision_resampler` has already pooled each frame down to
|
||||
# `pooled_grid_length ** 2` tokens.
|
||||
patch_tokens_per_frame = self.patch_grid_length**2
|
||||
if num_vision_tokens <= 0 or patch_tokens_per_frame <= 0:
|
||||
return 0
|
||||
if (
|
||||
num_mm_embeds <= 0
|
||||
or pooled_tokens_per_frame <= 0
|
||||
or patch_tokens_per_frame <= 0
|
||||
):
|
||||
return 0, 0
|
||||
|
||||
num_frames = num_vision_tokens // patch_tokens_per_frame
|
||||
return num_frames * (self.pooled_grid_length**2)
|
||||
num_frames = num_mm_embeds // pooled_tokens_per_frame
|
||||
return (
|
||||
num_frames * patch_tokens_per_frame,
|
||||
num_frames * pooled_tokens_per_frame,
|
||||
)
|
||||
|
||||
@@ -60,6 +60,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
|
||||
@@ -1281,21 +1282,18 @@ class Llama4ForConditionalGeneration(
|
||||
tower_model="vision_model.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
vision_config = self.config.vision_config
|
||||
patches_per_chunk = Mllama4ProcessingInfo.get_patch_per_chunk(vision_config)
|
||||
if num_image_tokens <= 0 or patches_per_chunk <= 0:
|
||||
return 0
|
||||
if num_mm_embeds <= 0 or patches_per_chunk <= 0:
|
||||
return 0, 0
|
||||
raw_patches = (vision_config.image_size // vision_config.patch_size) ** 2
|
||||
num_chunks = num_image_tokens // patches_per_chunk
|
||||
# Encoder processes raw_patches + 1 (CLS) per chunk
|
||||
return num_chunks * (raw_patches + 1)
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
vision_config = self.config.vision_config
|
||||
raw_patches = (vision_config.image_size // vision_config.patch_size) ** 2
|
||||
if num_vision_tokens <= 0:
|
||||
return 0
|
||||
num_chunks = num_vision_tokens // (raw_patches + 1)
|
||||
patches_per_chunk = Mllama4ProcessingInfo.get_patch_per_chunk(vision_config)
|
||||
return num_chunks * patches_per_chunk
|
||||
num_chunks = num_mm_embeds // patches_per_chunk
|
||||
return num_chunks * (raw_patches + 1), num_chunks * patches_per_chunk
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
@@ -1104,11 +1105,15 @@ class Moondream3ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
def get_language_model(self) -> nn.Module:
|
||||
return self.text
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
return num_image_tokens
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
return num_vision_tokens
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
return num_mm_embeds, num_mm_embeds
|
||||
|
||||
def _split_pixel_values(
|
||||
self,
|
||||
|
||||
@@ -24,7 +24,11 @@ from vllm.model_executor.models.interfaces import (
|
||||
SupportsPP,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
ImageEmbeddingItems,
|
||||
ImageProcessorItems,
|
||||
@@ -487,11 +491,15 @@ class OpenVLAForActionPrediction(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
tower_model="vision_backbone",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
return num_image_tokens
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
return num_vision_tokens
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
return num_mm_embeds, num_mm_embeds
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
def maybe_rename_vision_weights(
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -401,8 +402,12 @@ class PaliGemmaForConditionalGeneration(
|
||||
tower_model="vision_tower",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
return num_image_tokens
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
return num_vision_tokens
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
return num_mm_embeds, num_mm_embeds
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsOptionalItems,
|
||||
NestedTensors,
|
||||
)
|
||||
@@ -623,17 +624,18 @@ class PixtralForConditionalGeneration(
|
||||
tower_model="vision_encoder",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
if getattr(self, "patch_merger", None) is None:
|
||||
return num_image_tokens
|
||||
return num_mm_embeds, num_mm_embeds
|
||||
merge_size = self.vision_args.spatial_merge_size
|
||||
return num_image_tokens * (merge_size**2)
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
if getattr(self, "patch_merger", None) is None:
|
||||
return num_vision_tokens
|
||||
merge_size = self.vision_args.spatial_merge_size
|
||||
return num_vision_tokens // (merge_size**2)
|
||||
return num_mm_embeds * (merge_size**2), num_mm_embeds
|
||||
|
||||
|
||||
# Vision encoder
|
||||
|
||||
@@ -70,6 +70,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
@@ -2050,21 +2051,15 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
|
||||
return num_image_tokens * merge_size**2
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_vision_tokens // merge_size**2
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
@@ -66,6 +66,7 @@ from vllm.multimodal.inputs import (
|
||||
ImageItem,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
@@ -1747,21 +1748,15 @@ class Qwen2VLForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
|
||||
return num_image_tokens * merge_size**2
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_vision_tokens // merge_size**2
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
@@ -67,6 +67,7 @@ from vllm.multimodal.inputs import (
|
||||
AudioItem,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -625,16 +626,19 @@ class Qwen3ASRForConditionalGeneration(
|
||||
tower_model=["audio_tower."],
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int:
|
||||
"""Return the number of tokens processed by the audio tower encoder.
|
||||
|
||||
Required for LoRA support on the tower module.
|
||||
"""
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
# For Qwen3-ASR, the audio tower produces one embedding per audio
|
||||
# placeholder token inserted into the prompt (no additional
|
||||
# merge/downsample step like vision towers). Therefore, the encoder
|
||||
# token budget is identity.
|
||||
return num_audio_tokens
|
||||
return num_mm_embeds, None
|
||||
|
||||
@classmethod
|
||||
def get_speech_to_text_config(
|
||||
|
||||
@@ -3077,24 +3077,18 @@ class Qwen3VLForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
|
||||
return num_image_tokens * merge_size**2
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_vision_tokens // merge_size**2
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
NestedTensors,
|
||||
)
|
||||
@@ -373,9 +374,9 @@ def _build_chunk_attn_metadata(
|
||||
valid/padding boundary (equivalent to the key-padding mask the HF
|
||||
implementation uses), while every row still flows through the
|
||||
(potentially LoRA-wrapped) linears, keeping the per-chunk token counts
|
||||
constant as required by `get_num_mm_encoder_tokens` /
|
||||
`get_num_mm_connector_tokens`. Queries at padding positions produce
|
||||
(garbage) outputs, which are trimmed by `audio_token_len` downstream.
|
||||
constant as required by `get_mm_lora_token_counts`. Queries at padding
|
||||
positions produce (garbage) outputs, which are trimmed by `audio_token_len`
|
||||
downstream.
|
||||
"""
|
||||
batch_size = feature_lens.shape[0]
|
||||
starts = np.arange(batch_size, dtype=np.int64) * seq_len
|
||||
@@ -684,7 +685,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
|
||||
# LoRA on the tower/connector requires per-item token counts that are
|
||||
# predictable from the placeholder count alone (see
|
||||
# `get_num_mm_encoder_tokens`), so pad every audio chunk's mel input
|
||||
# `get_mm_lora_token_counts`), so pad every audio chunk's mel input
|
||||
# to the tower's full context instead of the batch's max length.
|
||||
self.pad_audio_to_max_context = bool(
|
||||
lora_config is not None and lora_config.enable_tower_connector_lora
|
||||
@@ -769,22 +770,26 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
self.config.audio_config.max_source_positions / self.config.stack_factor
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int:
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
# With `pad_audio_to_max_context` (enforced when tower/connector LoRA
|
||||
# is enabled), the tower's LoRA-wrapped linears always run on
|
||||
# `max_source_positions` conv-downsampled tokens per chunk, regardless
|
||||
# of the valid frame count.
|
||||
num_chunks = math.ceil(num_audio_tokens / self._get_max_tokens_per_chunk())
|
||||
return num_chunks * self.config.audio_config.max_source_positions
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_encoder_tokens: int) -> int:
|
||||
num_chunks = math.ceil(num_mm_embeds / self._get_max_tokens_per_chunk())
|
||||
tower_tokens = num_chunks * self.config.audio_config.max_source_positions
|
||||
# The connector runs on the frame-stacked tower output. Stacking pads
|
||||
# each chunk to a multiple of `stack_factor`, so this is
|
||||
# ceil(max_source_positions / stack_factor) tokens per chunk (188 for
|
||||
# whisper's 1500), not `num_encoder_tokens // stack_factor` (187).
|
||||
max_source_positions = self.config.audio_config.max_source_positions
|
||||
num_chunks = num_encoder_tokens // max_source_positions
|
||||
return num_chunks * self._get_max_tokens_per_chunk()
|
||||
# whisper's 1500), not `tower_tokens // stack_factor` (187).
|
||||
connector_tokens = num_chunks * self._get_max_tokens_per_chunk()
|
||||
return tower_tokens, connector_tokens
|
||||
|
||||
def _audio_features_to_embeddings(
|
||||
self,
|
||||
@@ -866,7 +871,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
# Pad every chunk to the tower's full context so the number of
|
||||
# tokens processed by the tower/connector per chunk is constant,
|
||||
# keeping the LoRA token mappings computed by
|
||||
# `get_num_mm_encoder_tokens`/`get_num_mm_connector_tokens` exact.
|
||||
# `get_mm_lora_token_counts` exact.
|
||||
# Attention to the extra padding is masked via `audio_lens`.
|
||||
# Over-long inputs are not trimmed here; the tower raises on them.
|
||||
pad_len = self.audio_tower.max_context_length - audio_features.shape[-1]
|
||||
|
||||
Reference in New Issue
Block a user