diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index f9d55cb97e..0ed8279629 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -571,7 +571,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Llama4ForConditionalGeneration` | Llama 4 | T + I+ | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + IE+ | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + IE+ | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
-| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + IE+ | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
+| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + IE+ | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | ✅︎ | ✅︎ |
| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | ✅︎ | ✅︎ |
| `LlavaOnevision2ForConditionalGeneration` | LLaVA-OneVision-2 | T + I+ + V+ | `lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct` | | |
| `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I+ + V+ | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
diff --git a/vllm/model_executor/models/llava_next.py b/vllm/model_executor/models/llava_next.py
index 44d50f434a..42a2df9824 100644
--- a/vllm/model_executor/models/llava_next.py
+++ b/vllm/model_executor/models/llava_next.py
@@ -15,13 +15,18 @@ from transformers.models.llava_next.modeling_llava_next import (
from vllm.config import VllmConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import MultiModalFieldConfig
+from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItem
from vllm.multimodal.parse import ImageSize
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .clip import CLIPVisionModel
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
+from .interfaces import (
+ MultiModalEmbeddings,
+ SupportsLoRA,
+ SupportsMultiModal,
+ SupportsPP,
+)
from .llava import (
BaseLlavaMultiModalProcessor,
BaseLlavaProcessingInfo,
@@ -30,6 +35,7 @@ from .llava import (
LlavaMultiModalProjector,
init_vision_tower_for_llava,
)
+from .module_mapping import MultiModelKeys
from .siglip import SiglipVisionModel
from .utils import (
AutoWeightsLoader,
@@ -37,7 +43,7 @@ from .utils import (
init_vllm_registered_model,
maybe_prefix,
)
-from .vision import get_num_selected_vision_tokens
+from .vision import get_num_selected_vision_tokens, get_vision_encoder_info
class LlavaNextImagePixelInputs(TensorSchema):
@@ -222,7 +228,14 @@ class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
info=LlavaNextProcessingInfo,
dummy_inputs=LlavaDummyInputsBuilder,
)
-class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
+class LlavaNextForConditionalGeneration(
+ nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
+):
+ packed_modules_mapping = {
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
+ "gate_up_proj": ["gate_proj", "up_proj"],
+ }
+
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
# mapping for new names in checkpoint saved after transformers v4.52
@@ -582,3 +595,38 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+
+ def get_mm_mapping(self) -> MultiModelKeys:
+ """
+ Get the module prefix in multimodal models
+ """
+ return MultiModelKeys.from_string_field(
+ language_model="language_model",
+ connector="multi_modal_projector",
+ tower_model="vision_tower",
+ )
+
+ def get_mm_lora_token_counts(
+ self,
+ *,
+ modality: str,
+ mm_kwargs: MultiModalKwargsItem | None,
+ num_mm_embeds: int,
+ ) -> tuple[int, int | None]:
+ del modality
+
+ pixel_values = mm_kwargs.get("pixel_values") if mm_kwargs else None
+ if pixel_values is None or not isinstance(pixel_values.data, torch.Tensor):
+ return num_mm_embeds, num_mm_embeds
+
+ # Unpad runs after the connector, so `num_mm_embeds` is not invertible.
+ num_tiles = pixel_values.data.shape[0]
+ encoder_info = get_vision_encoder_info(self.config)
+ tile_size = encoder_info.get_image_size()
+ tokens_per_tile = encoder_info.get_num_image_tokens(
+ image_width=tile_size, image_height=tile_size
+ )
+ selected_per_tile = get_num_selected_vision_tokens(
+ tokens_per_tile, self.config.vision_feature_select_strategy
+ )
+ return num_tiles * tokens_per_tile, num_tiles * selected_per_tile