From b6754f536e9cb40a1e117705a44d74218873b437 Mon Sep 17 00:00:00 2001 From: karthik <56480632+gangula-karthik@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:09:38 +0800 Subject: [PATCH] [Model] Enable LoRA support for tower and connector in LlavaNextVideo (#48594) Signed-off-by: gangula-karthik --- docs/models/supported_models.md | 2 +- .../processing/test_llava_next_video.py | 88 +++++++++++++++++++ .../model_executor/models/llava_next_video.py | 62 ++++++++++++- 3 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 tests/models/multimodal/processing/test_llava_next_video.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 37d27234aa..98f9cb5b65 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -555,7 +555,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + IE+ | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ | | `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + IE+ | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ | | `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + IE+ | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ | -| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ | +| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | ✅︎ | ✅︎ | | `LlavaOnevision2ForConditionalGeneration` | LLaVA-OneVision-2 | T + I+ + V+ | `lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct` | | | | `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I+ + V+ | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ | | `MiDashengLMModel` | MiDashengLM | T + A+ | `mispeech/midashenglm-7b` | | ✅︎ | diff --git a/tests/models/multimodal/processing/test_llava_next_video.py b/tests/models/multimodal/processing/test_llava_next_video.py new file mode 100644 index 0000000000..2dcbb7d4b5 --- /dev/null +++ b/tests/models/multimodal/processing/test_llava_next_video.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math + +import pytest + +from vllm.model_executor.models.llava_next_video import ( + LlavaNextVideoForConditionalGeneration, +) +from vllm.model_executor.models.vision import get_vision_encoder_info + +from ...utils import build_model_context + + +class _StubModel: + """Carries only the two attributes the token helpers read from + `self`, so the real methods can be exercised without constructing + the full `nn.Module` (vision tower, language model, etc.).""" + + patch_grid_length: int + pooled_grid_length: int + + +get_num_mm_encoder_tokens = ( + LlavaNextVideoForConditionalGeneration.get_num_mm_encoder_tokens +) +get_num_mm_connector_tokens = ( + LlavaNextVideoForConditionalGeneration.get_num_mm_connector_tokens +) + + +@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"]) +def test_num_mm_tokens_match_real_config(model_id): + """The stored grid lengths must match what `__init__` derives from + the real HF config, and the two helpers must invert each other's + frame-level scaling exactly.""" + ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1}) + config = ctx.model_config.hf_config + + vision_encoder_info = get_vision_encoder_info(config) + patch_grid_length = vision_encoder_info.get_patch_grid_length() + pooled_grid_length = math.ceil(patch_grid_length / config.spatial_pool_stride) + + stub = _StubModel() + stub.patch_grid_length = patch_grid_length + stub.pooled_grid_length = pooled_grid_length + + for num_frames in (1, 2, 8, 16, 32): + num_video_tokens = num_frames * pooled_grid_length**2 + + encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens) + assert encoder_tokens == num_frames * patch_grid_length**2 + + connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens) + assert connector_tokens == num_video_tokens + + +@pytest.mark.parametrize( + ("patch_grid_length", "pooled_grid_length", "num_frames"), + [ + (24, 12, 1), # llava-hf/LLaVA-NeXT-Video-7B-hf: 336 / 14, stride 2 + (24, 12, 16), + (27, 14, 5), # non-power-of-2 pooled grid (ceil rounding) + (16, 8, 6), + ], +) +def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_frames): + stub = _StubModel() + stub.patch_grid_length = patch_grid_length + stub.pooled_grid_length = pooled_grid_length + + num_video_tokens = num_frames * pooled_grid_length**2 + + encoder_tokens = get_num_mm_encoder_tokens(stub, num_video_tokens) + assert encoder_tokens == num_frames * patch_grid_length**2 + + connector_tokens = get_num_mm_connector_tokens(stub, encoder_tokens) + assert connector_tokens == num_video_tokens + + +def test_num_mm_tokens_zero(): + stub = _StubModel() + stub.patch_grid_length = 24 + stub.pooled_grid_length = 12 + + assert get_num_mm_encoder_tokens(stub, 0) == 0 + assert get_num_mm_connector_tokens(stub, 0) == 0 diff --git a/vllm/model_executor/models/llava_next_video.py b/vllm/model_executor/models/llava_next_video.py index e256768ef5..bb9c95017d 100644 --- a/vllm/model_executor/models/llava_next_video.py +++ b/vllm/model_executor/models/llava_next_video.py @@ -36,8 +36,14 @@ from vllm.sequence import IntermediateTensors from vllm.utils.collection_utils import is_list_of from vllm.utils.tensor_schema import TensorSchema, TensorShape -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP +from .interfaces import ( + MultiModalEmbeddings, + SupportsLoRA, + SupportsMultiModal, + SupportsPP, +) from .llava import init_vision_tower_for_llava +from .module_mapping import MultiModelKeys from .siglip import SiglipVisionModel from .utils import ( AutoWeightsLoader, @@ -298,7 +304,14 @@ class LlavaNextMultiModalProjector(nn.Module): info=LlavaNextVideoProcessingInfo, dummy_inputs=LlavaNextVideoDummyInputsBuilder, ) -class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): +class LlavaNextVideoForConditionalGeneration( + nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP +): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + } + hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ # mapping for new names in checkpoint saved after transformers v4.52 @@ -327,6 +340,12 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp self.config = config self.multimodal_config = multimodal_config + vision_encoder_info = get_vision_encoder_info(config) + self.patch_grid_length = vision_encoder_info.get_patch_grid_length() + self.pooled_grid_length = math.ceil( + self.patch_grid_length / config.spatial_pool_stride + ) + with self._mark_tower_model(vllm_config, "video"): # Initialize the vision tower only up to the required feature layer self.vision_tower = init_vision_tower_for_llava( @@ -460,3 +479,42 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp ignore_unexpected_prefixes=["image_newline"], ) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + + def get_mm_mapping(self) -> MultiModelKeys: + """ + Get the module prefix in multimodal models + """ + return MultiModelKeys.from_string_field( + language_model="language_model", + connector="multi_modal_projector", + tower_model="vision_tower", + ) + + def get_num_mm_encoder_tokens( + self, + num_video_tokens: int, + ) -> int: + # Invert the spatial pooling done by `vision_resampler`: each frame + # contributes `pooled_grid_length ** 2` tokens after the language + # model's placeholder count, but `patch_grid_length ** 2` tokens + # when it leaves the vision encoder. + pooled_tokens_per_frame = self.pooled_grid_length**2 + if num_video_tokens <= 0 or pooled_tokens_per_frame <= 0: + return 0 + + num_frames = num_video_tokens // pooled_tokens_per_frame + return num_frames * (self.patch_grid_length**2) + + def get_num_mm_connector_tokens( + self, + num_vision_tokens: int, + ) -> int: + # The projector is length-preserving; it runs after + # `vision_resampler` has already pooled each frame down to + # `pooled_grid_length ** 2` tokens. + patch_tokens_per_frame = self.patch_grid_length**2 + if num_vision_tokens <= 0 or patch_tokens_per_frame <= 0: + return 0 + + num_frames = num_vision_tokens // patch_tokens_per_frame + return num_frames * (self.pooled_grid_length**2)