[MM] Enable device normalization for Llama Nemotron VL Embed/Rerank (#57928)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
61349e342d
commit
70dc122f81
@@ -8,6 +8,10 @@ import torch.nn as nn
|
||||
from transformers import AutoModel, PreTrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
FusedMMInputNorm,
|
||||
IdentityInputNorm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -26,6 +30,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import cached_image_processor_from_config
|
||||
from vllm.transformers_utils.processors.nemotron_vl import (
|
||||
SIGLIP_MEAN,
|
||||
SIGLIP_STD,
|
||||
LlamaNemotronNanoVLImageProcessor,
|
||||
LlamaNemotronNanoVLProcessor,
|
||||
LlamaNemotronVLEmbedImageProcessor,
|
||||
@@ -474,6 +480,7 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
# Weight mapping from checkpoint format to vLLM format
|
||||
# Different from parent class due to different vision model structure
|
||||
@@ -498,6 +505,15 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
|
||||
# Override: get img_context_token_id from config (parent sets None)
|
||||
self.img_context_token_id = getattr(config, "img_context_token_id", None)
|
||||
|
||||
if self.model_config.get_multimodal_config().mm_device_do_normalize:
|
||||
self.input_norm = FusedMMInputNorm(
|
||||
image_mean=list(SIGLIP_MEAN),
|
||||
image_std=list(SIGLIP_STD),
|
||||
rescale_factor=1 / 255,
|
||||
)
|
||||
else:
|
||||
self.input_norm = IdentityInputNorm()
|
||||
|
||||
# Initialize pooler for embedding output
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
@@ -537,6 +553,10 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
|
||||
|
||||
def _call_vision_model(self, pixel_values: torch.Tensor) -> torch.Tensor:
|
||||
"""Override to handle SigLIP interface."""
|
||||
original_shape = pixel_values.shape
|
||||
pixel_values = self.input_norm(
|
||||
pixel_values.flatten(start_dim=1), self.vision_model.dtype
|
||||
).view(original_shape)
|
||||
return self.vision_model(pixel_values)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
Reference in New Issue
Block a user