[MM][Mistral3] Image preprocessing optimization (#57531)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
65ed0a2642
commit
d86cf91240
@@ -147,6 +147,11 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams:
|
||||
rescale_factor)`` from the processor config."""
|
||||
config = get_processor_config(model_config.model, revision=model_config.revision)
|
||||
config = config.get("image_processor", config)
|
||||
if not any(
|
||||
key in config
|
||||
for key in ("do_normalize", "do_rescale", "image_mean", "image_std")
|
||||
):
|
||||
config = model_config.hf_image_processor_config
|
||||
|
||||
has_norm_params = "image_mean" in config and "image_std" in config
|
||||
do_normalize = bool(config.get("do_normalize", has_norm_params))
|
||||
|
||||
@@ -115,6 +115,8 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
|
||||
dummy_inputs=Mistral3DummyInputsBuilder,
|
||||
)
|
||||
class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
|
||||
supports_mm_device_do_normalize = False
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.vision_encoder.": "vision_tower.",
|
||||
|
||||
@@ -7,6 +7,8 @@ from typing import Annotated, Any, Literal, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
|
||||
from transformers.models.pixtral import PixtralProcessor
|
||||
|
||||
@@ -14,6 +16,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -56,6 +59,10 @@ from .utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version(
|
||||
"5.15.0"
|
||||
)
|
||||
|
||||
|
||||
class Mistral3ImagePixelInputs(TensorSchema):
|
||||
"""Dimensions:
|
||||
@@ -225,7 +232,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
|
||||
image_size = size["longest_edge"]
|
||||
return Mistral3HFEncoderInfo(self.get_hf_config(), image_size)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
def get_hf_processor(self, **kwargs: object) -> PixtralProcessor:
|
||||
return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
@@ -277,8 +284,12 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
|
||||
|
||||
|
||||
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
# PixtralProcessor supports image-only calls starting in transformers 5.15.
|
||||
# Older releases need dummy text and take the slower tokenization path.
|
||||
if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
return None
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
@@ -378,6 +389,7 @@ def init_vision_tower_for_mistral3(
|
||||
hf_config: Mistral3Config,
|
||||
quant_config: QuantizationConfig | None,
|
||||
*,
|
||||
input_norm: nn.Module | None = None,
|
||||
require_post_norm: bool | None = None,
|
||||
prefix: str = "",
|
||||
) -> PixtralHFVisionModel:
|
||||
@@ -391,6 +403,7 @@ def init_vision_tower_for_mistral3(
|
||||
return PixtralHFVisionModel(
|
||||
vision_config,
|
||||
quant_config=quant_config,
|
||||
input_norm=input_norm,
|
||||
num_hidden_layers_override=num_hidden_layers,
|
||||
require_post_norm=require_post_norm,
|
||||
prefix=prefix,
|
||||
@@ -410,6 +423,8 @@ class Mistral3ForConditionalGeneration(
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
):
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
@@ -464,6 +479,7 @@ class Mistral3ForConditionalGeneration(
|
||||
self.vision_tower = init_vision_tower_for_mistral3(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(vllm_config.model_config),
|
||||
require_post_norm=False,
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Annotated, Literal
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
|
||||
from mistral_common.protocol.instruct.messages import UserMessage
|
||||
from mistral_common.protocol.instruct.request import ChatCompletionRequest
|
||||
@@ -1427,12 +1428,55 @@ class PixtralHFTransformer(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def pixtral_patch_embed(
|
||||
images: list[torch.Tensor],
|
||||
weight: torch.Tensor,
|
||||
input_norm: nn.Module,
|
||||
) -> tuple[torch.Tensor, list[torch.Tensor]]:
|
||||
"""Normalize and project all CHW images in one matrix multiplication."""
|
||||
out_channels, in_channels, patch_height, patch_width = weight.shape
|
||||
grid_sizes = []
|
||||
for image in images:
|
||||
assert image.ndim == 3 and image.shape[0] == in_channels
|
||||
_, height, width = image.shape
|
||||
patch_rows = height // patch_height
|
||||
patch_cols = width // patch_width
|
||||
grid_sizes.append((patch_rows, patch_cols))
|
||||
|
||||
patch_counts = [rows * cols for rows, cols in grid_sizes]
|
||||
packed = torch.empty(
|
||||
(sum(patch_counts), in_channels * patch_height * patch_width),
|
||||
dtype=images[0].dtype,
|
||||
device=images[0].device,
|
||||
)
|
||||
offset = 0
|
||||
for image, (rows, cols), count in zip(images, grid_sizes, patch_counts):
|
||||
source = (
|
||||
image[:, : rows * patch_height, : cols * patch_width]
|
||||
.reshape(in_channels, rows, patch_height, cols, patch_width)
|
||||
.permute(1, 3, 0, 2, 4)
|
||||
)
|
||||
packed[offset : offset + count].view(
|
||||
rows, cols, in_channels, patch_height, patch_width
|
||||
).copy_(source)
|
||||
offset += count
|
||||
|
||||
normalized = input_norm(packed, weight.dtype)
|
||||
projected = F.linear(normalized, weight.reshape(out_channels, -1))
|
||||
embeddings = [
|
||||
tokens.transpose(0, 1).reshape(1, out_channels, rows, cols)
|
||||
for tokens, (rows, cols) in zip(projected.split(patch_counts), grid_sizes)
|
||||
]
|
||||
return projected.unsqueeze(0), embeddings
|
||||
|
||||
|
||||
class PixtralHFVisionModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PixtralVisionConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
*,
|
||||
input_norm: nn.Module | None = None,
|
||||
num_hidden_layers_override: int | None = None,
|
||||
require_post_norm: bool | None = None,
|
||||
prefix: str = "",
|
||||
@@ -1440,6 +1484,7 @@ class PixtralHFVisionModel(nn.Module):
|
||||
super().__init__()
|
||||
|
||||
self.config = config
|
||||
self.input_norm = input_norm
|
||||
|
||||
self.patch_conv = Conv2dLayer(
|
||||
in_channels=config.num_channels,
|
||||
@@ -1495,16 +1540,24 @@ class PixtralHFVisionModel(nn.Module):
|
||||
all tokens of all images of shape (N_toks, D)
|
||||
|
||||
"""
|
||||
# pass images through initial convolution independently
|
||||
patch_embeds_list = [
|
||||
self.patch_conv(img.unsqueeze(0).to(self.dtype)) for img in pixel_values
|
||||
]
|
||||
images = [image.to(device=self.device) for image in pixel_values]
|
||||
if self.input_norm is not None:
|
||||
patch_embeds, patch_embeds_list = pixtral_patch_embed(
|
||||
images,
|
||||
self.patch_conv.weight,
|
||||
self.input_norm,
|
||||
)
|
||||
else:
|
||||
patch_embeds_list = []
|
||||
for image in images:
|
||||
patch_embeds_list.append(
|
||||
self.patch_conv(image.unsqueeze(0).to(self.dtype))
|
||||
)
|
||||
patch_embeds = torch.cat(
|
||||
[p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1
|
||||
)
|
||||
|
||||
patch_embeds = [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list]
|
||||
embed_sizes = [p.shape[1] for p in patch_embeds]
|
||||
|
||||
# flatten to a single sequence
|
||||
patch_embeds = torch.cat(patch_embeds, dim=1)
|
||||
embed_sizes = [p.shape[-2] * p.shape[-1] for p in patch_embeds_list]
|
||||
patch_embeds = self.ln_pre(patch_embeds)
|
||||
|
||||
# positional embeddings
|
||||
|
||||
Reference in New Issue
Block a user