[MM][Mistral3] Image preprocessing optimization (#57531)

Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jakub Zakrzewski
2026-09-29 06:53:38 +00:00
committed by GitHub
co-authored by OpenAI Codex OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 65ed0a2642
commit d86cf91240
6 changed files with 228 additions and 25 deletions
@@ -4,14 +4,29 @@
import pytest
import torch
import torch.nn.functional as F
from PIL import Image
from transformers import AutoProcessor, BatchFeature
from transformers.models.pixtral import PixtralProcessor
from vllm.model_executor.models.lightonocr import LightOnOCRProcessingInfo
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.models.lightonocr import (
LightOnOCRForConditionalGeneration,
LightOnOCRProcessingInfo,
)
from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo
from vllm.model_executor.models.pixtral import PixtralHFEncoderInfo
from vllm.model_executor.models.pixtral import (
PixtralHFEncoderInfo,
pixtral_patch_embed,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItems
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from ...utils import build_model_context
@@ -21,6 +36,65 @@ _MODEL_CONFIG_KWARGS = {"config_format": "hf"}
_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
_RGB_RESCALE = 1.0 / 255.0
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.skipif(
not HAS_TRITON or not current_platform.is_cuda(), reason="CUDA Triton kernel"
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("normalize", [False, True])
@pytest.mark.parametrize(("channels", "patch_shape"), [(3, (4, 4)), (2, (2, 4))])
def test_pixtral_patch_embed_preserves_image_order_and_strides(
dtype: torch.dtype,
normalize: bool,
channels: int,
patch_shape: tuple[int, int],
):
"""Packed projection matches CHW normalization and patch convolution."""
device = torch.device(current_platform.device_type)
norm = (
FusedMMInputNorm(
_RGB_MEAN[:channels],
_RGB_STD[:channels],
_RGB_RESCALE,
channel=channels,
).to(device)
if normalize
else IdentityInputNorm()
)
weight = torch.randn(32, channels, *patch_shape, device=device, dtype=dtype)
images = [
torch.randint(0, 256, (channels, 17, 18), device=device, dtype=torch.uint8),
torch.randint(0, 256, (channels, 25, 17), device=device, dtype=torch.uint8)[
:, 1:17, 1:17
],
torch.randint(0, 256, (channels, 19, 20), device=device, dtype=torch.uint8)[
:, 1:17, 2:18
],
]
reference = []
for image in images:
if normalize:
normalized = (
image.float() * norm.weight[:, None, None] + norm.bias[:, None, None]
)
else:
normalized = image
reference.append(
F.conv2d(normalized.to(dtype).unsqueeze(0), weight, stride=patch_shape)
)
packed, actual = pixtral_patch_embed(images, weight, norm)
for expected, result in zip(reference, actual):
torch.testing.assert_close(result, expected, rtol=0.01, atol=0.02)
expected_packed = torch.cat(
[result.flatten(2).transpose(1, 2) for result in reference], dim=1
)
torch.testing.assert_close(packed, expected_packed, rtol=0.01, atol=0.02)
def _process_images_with_hf(
hf_processor,
@@ -128,6 +202,9 @@ def test_processor_size_override(
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
vllm_hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
assert isinstance(vllm_hf_processor, PixtralProcessor)
hf_processor = AutoProcessor.from_pretrained(
model_id,
fix_mistral_regex=True,
@@ -153,6 +230,53 @@ def test_processor_size_override(
assert prompt_update_tokens == hf_placeholder_tokens
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
ctx = build_model_context(
_MODEL_ID,
limit_mm_per_prompt={"image": 2},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (31, 47), color=(17, 89, 231)),
Image.new("RGB", (48, 32), color=(201, 13, 127)),
]
prompt = [processor.info.get_hf_config().image_token_index] * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert all(value.dtype == torch.uint8 for value in raw_values)
input_norm = build_mm_input_norm(ctx.model_config)
patch_size = processor.info.get_hf_config().vision_config.patch_size
def pack_patches(image: torch.Tensor) -> torch.Tensor:
channels, height, width = image.shape
rows = height // patch_size
cols = width // patch_size
return (
image[:, : rows * patch_size, : cols * patch_size]
.reshape(channels, rows, patch_size, cols, patch_size)
.permute(1, 3, 0, 2, 4)
.reshape(-1, channels * patch_size**2)
)
for raw, normalized in zip(raw_values, normalized_values):
output = input_norm(pack_patches(raw), normalized.dtype)
torch.testing.assert_close(
output, pack_patches(normalized), rtol=1e-5, atol=1e-6
)
def test_scoped_request_size_overrides_configured_flat_size():
ctx = build_model_context(
_MODEL_ID,
@@ -188,6 +312,8 @@ def test_scoped_request_size_overrides_configured_flat_size():
def test_lightonocr_keeps_vision_config_image_size():
assert not LightOnOCRForConditionalGeneration.supports_mm_device_do_normalize
ctx = build_model_context(
_LIGHTON_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},