[Model] Enable LoRA support for tower and connector in LlavaNextVideo (#48594)
Signed-off-by: gangula-karthik <gkarthik923@gmail.com>
This commit is contained in:
@@ -555,7 +555,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
|
||||
| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
|
||||
| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
|
||||
| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ |
|
||||
| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | ✅︎ | ✅︎ |
|
||||
| `LlavaOnevision2ForConditionalGeneration` | LLaVA-OneVision-2 | T + I<sup>+</sup> + V<sup>+</sup> | `lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct` | | |
|
||||
| `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I<sup>+</sup> + V<sup>+</sup> | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
|
||||
| `MiDashengLMModel` | MiDashengLM | T + A<sup>+</sup> | `mispeech/midashenglm-7b` | | ✅︎ |
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.llava_next_video import (
|
||||
LlavaNextVideoForConditionalGeneration,
|
||||
)
|
||||
from vllm.model_executor.models.vision import get_vision_encoder_info
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
class _StubModel:
|
||||
"""Carries only the two attributes the token helpers read from
|
||||
`self`, so the real methods can be exercised without constructing
|
||||
the full `nn.Module` (vision tower, language model, etc.)."""
|
||||
|
||||
patch_grid_length: int
|
||||
pooled_grid_length: int
|
||||
|
||||
|
||||
get_num_mm_encoder_tokens = (
|
||||
LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens
|
||||
)
|
||||
get_num_mm_connector_tokens = (
|
||||
LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
|
||||
def test_num_mm_tokens_match_real_config(model_id):
|
||||
"""The stored grid lengths must match what `__init__` derives from
|
||||
the real HF config, and the two helpers must invert each other's
|
||||
frame-level scaling exactly."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
|
||||
config = ctx.model_config.hf_config
|
||||
|
||||
vision_encoder_info = get_vision_encoder_info(config)
|
||||
patch_grid_length = vision_encoder_info.get_patch_grid_length()
|
||||
pooled_grid_length = math.ceil(patch_grid_length / config.spatial_pool_stride)
|
||||
|
||||
stub = _StubModel()
|
||||
stub.patch_grid_length = patch_grid_length
|
||||
stub.pooled_grid_length = pooled_grid_length
|
||||
|
||||
for num_frames in (1, 2, 8, 16, 32):
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
|
||||
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("patch_grid_length", "pooled_grid_length", "num_frames"),
|
||||
[
|
||||
(24, 12, 1), # llava-hf/LLaVA-NeXT-Video-7B-hf: 336 / 14, stride 2
|
||||
(24, 12, 16),
|
||||
(27, 14, 5), # non-power-of-2 pooled grid (ceil rounding)
|
||||
(16, 8, 6),
|
||||
],
|
||||
)
|
||||
def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_frames):
|
||||
stub = _StubModel()
|
||||
stub.patch_grid_length = patch_grid_length
|
||||
stub.pooled_grid_length = pooled_grid_length
|
||||
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
|
||||
connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens)
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
def test_num_mm_tokens_zero():
|
||||
stub = _StubModel()
|
||||
stub.patch_grid_length = 24
|
||||
stub.pooled_grid_length = 12
|
||||
|
||||
assert get_num_mm_encoder_tokens(stub, 0) == 0
|
||||
assert get_num_mm_connector_tokens(stub, 0) == 0
|
||||
@@ -36,8 +36,14 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from .llava import init_vision_tower_for_llava
|
||||
from .module_mapping import MultiModelKeys
|
||||
from .siglip import SiglipVisionModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -298,7 +304,14 @@ class LlavaNextMultiModalProjector(nn.Module):
|
||||
info=LlavaNextVideoProcessingInfo,
|
||||
dummy_inputs=LlavaNextVideoDummyInputsBuilder,
|
||||
)
|
||||
class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
class LlavaNextVideoForConditionalGeneration(
|
||||
nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
|
||||
):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
# mapping for new names in checkpoint saved after transformers v4.52
|
||||
@@ -327,6 +340,12 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
self.config = config
|
||||
self.multimodal_config = multimodal_config
|
||||
|
||||
vision_encoder_info = get_vision_encoder_info(config)
|
||||
self.patch_grid_length = vision_encoder_info.get_patch_grid_length()
|
||||
self.pooled_grid_length = math.ceil(
|
||||
self.patch_grid_length / config.spatial_pool_stride
|
||||
)
|
||||
|
||||
with self._mark_tower_model(vllm_config, "video"):
|
||||
# Initialize the vision tower only up to the required feature layer
|
||||
self.vision_tower = init_vision_tower_for_llava(
|
||||
@@ -460,3 +479,42 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
ignore_unexpected_prefixes=["image_newline"],
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
tower_model="vision_tower",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
self,
|
||||
num_video_tokens: int,
|
||||
) -> int:
|
||||
# Invert the spatial pooling done by `vision_resampler`: each frame
|
||||
# contributes `pooled_grid_length ** 2` tokens after the language
|
||||
# model's placeholder count, but `patch_grid_length ** 2` tokens
|
||||
# when it leaves the vision encoder.
|
||||
pooled_tokens_per_frame = self.pooled_grid_length**2
|
||||
if num_video_tokens <= 0 or pooled_tokens_per_frame <= 0:
|
||||
return 0
|
||||
|
||||
num_frames = num_video_tokens // pooled_tokens_per_frame
|
||||
return num_frames * (self.patch_grid_length**2)
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
# The projector is length-preserving; it runs after
|
||||
# `vision_resampler` has already pooled each frame down to
|
||||
# `pooled_grid_length ** 2` tokens.
|
||||
patch_tokens_per_frame = self.patch_grid_length**2
|
||||
if num_vision_tokens <= 0 or patch_tokens_per_frame <= 0:
|
||||
return 0
|
||||
|
||||
num_frames = num_vision_tokens // patch_tokens_per_frame
|
||||
return num_frames * (self.pooled_grid_length**2)
|
||||
|
||||
Reference in New Issue
Block a user