[MM][Mistral3] Image preprocessing optimization (#57531)

Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jakub Zakrzewski
2026-09-29 06:53:38 +00:00
committed by GitHub
co-authored by OpenAI Codex OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 65ed0a2642
commit d86cf91240
6 changed files with 228 additions and 25 deletions
+19 -3
View File
@@ -7,6 +7,8 @@ from typing import Annotated, Any, Literal, cast
import torch
import torch.nn as nn
import transformers
from packaging.version import Version
from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
from transformers.models.pixtral import PixtralProcessor
@@ -14,6 +16,7 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -56,6 +59,10 @@ from .utils import (
maybe_prefix,
)
TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version(
"5.15.0"
)
class Mistral3ImagePixelInputs(TensorSchema):
"""Dimensions:
@@ -225,7 +232,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
image_size = size["longest_edge"]
return Mistral3HFEncoderInfo(self.get_hf_config(), image_size)
def get_hf_processor(self, **kwargs: object):
def get_hf_processor(self, **kwargs: object) -> PixtralProcessor:
return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -277,8 +284,12 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
# PixtralProcessor supports image-only calls starting in transformers 5.15.
# Older releases need dummy text and take the slower tokenization path.
if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY:
return self.dummy_inputs.get_dummy_text(mm_counts)
return None
def _postprocess_hf_mm_data(
self,
@@ -378,6 +389,7 @@ def init_vision_tower_for_mistral3(
hf_config: Mistral3Config,
quant_config: QuantizationConfig | None,
*,
input_norm: nn.Module | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
) -> PixtralHFVisionModel:
@@ -391,6 +403,7 @@ def init_vision_tower_for_mistral3(
return PixtralHFVisionModel(
vision_config,
quant_config=quant_config,
input_norm=input_norm,
num_hidden_layers_override=num_hidden_layers,
require_post_norm=require_post_norm,
prefix=prefix,
@@ -410,6 +423,8 @@ class Mistral3ForConditionalGeneration(
SupportsEagle,
SupportsEagle3,
):
supports_mm_device_do_normalize = True
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -464,6 +479,7 @@ class Mistral3ForConditionalGeneration(
self.vision_tower = init_vision_tower_for_mistral3(
config,
quant_config=quant_config,
input_norm=build_mm_input_norm(vllm_config.model_config),
require_post_norm=False,
prefix=maybe_prefix(prefix, "vision_tower"),
)