[MM] Enable device normalization for Llama Nemotron VL Embed/Rerank (#57928)

Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Jakub Zakrzewski
2026-09-29 08:33:52 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 61349e342d
commit 70dc122f81
4 changed files with 132 additions and 20 deletions
+20
View File
@@ -8,6 +8,10 @@ import torch.nn as nn
from transformers import AutoModel, PreTrainedConfig
from vllm.config import VllmConfig
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
)
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -26,6 +30,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_image_processor_from_config
from vllm.transformers_utils.processors.nemotron_vl import (
SIGLIP_MEAN,
SIGLIP_STD,
LlamaNemotronNanoVLImageProcessor,
LlamaNemotronNanoVLProcessor,
LlamaNemotronVLEmbedImageProcessor,
@@ -474,6 +480,7 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
"""
is_pooling_model = True
supports_mm_device_do_normalize = True
# Weight mapping from checkpoint format to vLLM format
# Different from parent class due to different vision model structure
@@ -498,6 +505,15 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
# Override: get img_context_token_id from config (parent sets None)
self.img_context_token_id = getattr(config, "img_context_token_id", None)
if self.model_config.get_multimodal_config().mm_device_do_normalize:
self.input_norm = FusedMMInputNorm(
image_mean=list(SIGLIP_MEAN),
image_std=list(SIGLIP_STD),
rescale_factor=1 / 255,
)
else:
self.input_norm = IdentityInputNorm()
# Initialize pooler for embedding output
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
@@ -537,6 +553,10 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
def _call_vision_model(self, pixel_values: torch.Tensor) -> torch.Tensor:
"""Override to handle SigLIP interface."""
original_shape = pixel_values.shape
pixel_values = self.input_norm(
pixel_values.flatten(start_dim=1), self.vision_model.dtype
).view(original_shape)
return self.vision_model(pixel_values)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: