[Model] Enable LoRA support for tower and connector in LlavaNextForConditionalGeneration (#49788)
Signed-off-by: gangula-karthik <gkarthik923@gmail.com>
This commit is contained in:
@@ -571,7 +571,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `Llama4ForConditionalGeneration` | Llama 4 | T + I<sup>+</sup> | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
|
||||
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
|
||||
| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
|
||||
| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
|
||||
| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | ✅︎ | ✅︎ |
|
||||
| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | ✅︎ | ✅︎ |
|
||||
| `LlavaOnevision2ForConditionalGeneration` | LLaVA-OneVision-2 | T + I<sup>+</sup> + V<sup>+</sup> | `lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct` | | |
|
||||
| `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I<sup>+</sup> + V<sup>+</sup> | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
|
||||
|
||||
@@ -15,13 +15,18 @@ from transformers.models.llava_next.modeling_llava_next import (
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItem
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .clip import CLIPVisionModel
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from .llava import (
|
||||
BaseLlavaMultiModalProcessor,
|
||||
BaseLlavaProcessingInfo,
|
||||
@@ -30,6 +35,7 @@ from .llava import (
|
||||
LlavaMultiModalProjector,
|
||||
init_vision_tower_for_llava,
|
||||
)
|
||||
from .module_mapping import MultiModelKeys
|
||||
from .siglip import SiglipVisionModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -37,7 +43,7 @@ from .utils import (
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import get_num_selected_vision_tokens
|
||||
from .vision import get_num_selected_vision_tokens, get_vision_encoder_info
|
||||
|
||||
|
||||
class LlavaNextImagePixelInputs(TensorSchema):
|
||||
@@ -222,7 +228,14 @@ class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):
|
||||
info=LlavaNextProcessingInfo,
|
||||
dummy_inputs=LlavaDummyInputsBuilder,
|
||||
)
|
||||
class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
class LlavaNextForConditionalGeneration(
|
||||
nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP
|
||||
):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
# mapping for new names in checkpoint saved after transformers v4.52
|
||||
@@ -582,3 +595,38 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
tower_model="vision_tower",
|
||||
)
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality
|
||||
|
||||
pixel_values = mm_kwargs.get("pixel_values") if mm_kwargs else None
|
||||
if pixel_values is None or not isinstance(pixel_values.data, torch.Tensor):
|
||||
return num_mm_embeds, num_mm_embeds
|
||||
|
||||
# Unpad runs after the connector, so `num_mm_embeds` is not invertible.
|
||||
num_tiles = pixel_values.data.shape[0]
|
||||
encoder_info = get_vision_encoder_info(self.config)
|
||||
tile_size = encoder_info.get_image_size()
|
||||
tokens_per_tile = encoder_info.get_num_image_tokens(
|
||||
image_width=tile_size, image_height=tile_size
|
||||
)
|
||||
selected_per_tile = get_num_selected_vision_tokens(
|
||||
tokens_per_tile, self.config.vision_feature_select_strategy
|
||||
)
|
||||
return num_tiles * tokens_per_tile, num_tiles * selected_per_tile
|
||||
|
||||
Reference in New Issue
Block a user