[MM][Mistral3] Image preprocessing optimization (#57531)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
65ed0a2642
commit
d86cf91240
@@ -4,14 +4,29 @@
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from transformers import AutoProcessor, BatchFeature
|
||||
from transformers.models.pixtral import PixtralProcessor
|
||||
|
||||
from vllm.model_executor.models.lightonocr import LightOnOCRProcessingInfo
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
FusedMMInputNorm,
|
||||
IdentityInputNorm,
|
||||
build_mm_input_norm,
|
||||
)
|
||||
from vllm.model_executor.models.lightonocr import (
|
||||
LightOnOCRForConditionalGeneration,
|
||||
LightOnOCRProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo
|
||||
from vllm.model_executor.models.pixtral import PixtralHFEncoderInfo
|
||||
from vllm.model_executor.models.pixtral import (
|
||||
PixtralHFEncoderInfo,
|
||||
pixtral_patch_embed,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItems
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
@@ -21,6 +36,65 @@ _MODEL_CONFIG_KWARGS = {"config_format": "hf"}
|
||||
_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
|
||||
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
|
||||
|
||||
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
|
||||
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
|
||||
_RGB_RESCALE = 1.0 / 255.0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TRITON or not current_platform.is_cuda(), reason="CUDA Triton kernel"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("normalize", [False, True])
|
||||
@pytest.mark.parametrize(("channels", "patch_shape"), [(3, (4, 4)), (2, (2, 4))])
|
||||
def test_pixtral_patch_embed_preserves_image_order_and_strides(
|
||||
dtype: torch.dtype,
|
||||
normalize: bool,
|
||||
channels: int,
|
||||
patch_shape: tuple[int, int],
|
||||
):
|
||||
"""Packed projection matches CHW normalization and patch convolution."""
|
||||
device = torch.device(current_platform.device_type)
|
||||
norm = (
|
||||
FusedMMInputNorm(
|
||||
_RGB_MEAN[:channels],
|
||||
_RGB_STD[:channels],
|
||||
_RGB_RESCALE,
|
||||
channel=channels,
|
||||
).to(device)
|
||||
if normalize
|
||||
else IdentityInputNorm()
|
||||
)
|
||||
weight = torch.randn(32, channels, *patch_shape, device=device, dtype=dtype)
|
||||
images = [
|
||||
torch.randint(0, 256, (channels, 17, 18), device=device, dtype=torch.uint8),
|
||||
torch.randint(0, 256, (channels, 25, 17), device=device, dtype=torch.uint8)[
|
||||
:, 1:17, 1:17
|
||||
],
|
||||
torch.randint(0, 256, (channels, 19, 20), device=device, dtype=torch.uint8)[
|
||||
:, 1:17, 2:18
|
||||
],
|
||||
]
|
||||
reference = []
|
||||
for image in images:
|
||||
if normalize:
|
||||
normalized = (
|
||||
image.float() * norm.weight[:, None, None] + norm.bias[:, None, None]
|
||||
)
|
||||
else:
|
||||
normalized = image
|
||||
reference.append(
|
||||
F.conv2d(normalized.to(dtype).unsqueeze(0), weight, stride=patch_shape)
|
||||
)
|
||||
packed, actual = pixtral_patch_embed(images, weight, norm)
|
||||
for expected, result in zip(reference, actual):
|
||||
torch.testing.assert_close(result, expected, rtol=0.01, atol=0.02)
|
||||
expected_packed = torch.cat(
|
||||
[result.flatten(2).transpose(1, 2) for result in reference], dim=1
|
||||
)
|
||||
torch.testing.assert_close(packed, expected_packed, rtol=0.01, atol=0.02)
|
||||
|
||||
|
||||
def _process_images_with_hf(
|
||||
hf_processor,
|
||||
@@ -128,6 +202,9 @@ def test_processor_size_override(
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
vllm_hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
assert isinstance(vllm_hf_processor, PixtralProcessor)
|
||||
|
||||
hf_processor = AutoProcessor.from_pretrained(
|
||||
model_id,
|
||||
fix_mistral_regex=True,
|
||||
@@ -153,6 +230,53 @@ def test_processor_size_override(
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (31, 47), color=(17, 89, 231)),
|
||||
Image.new("RGB", (48, 32), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = [processor.info.get_hf_config().image_token_index] * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert all(value.dtype == torch.uint8 for value in raw_values)
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config)
|
||||
patch_size = processor.info.get_hf_config().vision_config.patch_size
|
||||
|
||||
def pack_patches(image: torch.Tensor) -> torch.Tensor:
|
||||
channels, height, width = image.shape
|
||||
rows = height // patch_size
|
||||
cols = width // patch_size
|
||||
return (
|
||||
image[:, : rows * patch_size, : cols * patch_size]
|
||||
.reshape(channels, rows, patch_size, cols, patch_size)
|
||||
.permute(1, 3, 0, 2, 4)
|
||||
.reshape(-1, channels * patch_size**2)
|
||||
)
|
||||
|
||||
for raw, normalized in zip(raw_values, normalized_values):
|
||||
output = input_norm(pack_patches(raw), normalized.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, pack_patches(normalized), rtol=1e-5, atol=1e-6
|
||||
)
|
||||
|
||||
|
||||
def test_scoped_request_size_overrides_configured_flat_size():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
@@ -188,6 +312,8 @@ def test_scoped_request_size_overrides_configured_flat_size():
|
||||
|
||||
|
||||
def test_lightonocr_keeps_vision_config_image_size():
|
||||
assert not LightOnOCRForConditionalGeneration.supports_mm_device_do_normalize
|
||||
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
mm_processor_kwargs={"size": {"longest_edge": 1008}},
|
||||
|
||||
Reference in New Issue
Block a user