diff --git a/tests/models/multimodal/processing/test_mistral3.py b/tests/models/multimodal/processing/test_mistral3.py index fcb783f22f..4d900e627f 100644 --- a/tests/models/multimodal/processing/test_mistral3.py +++ b/tests/models/multimodal/processing/test_mistral3.py @@ -5,7 +5,7 @@ import pytest import torch from PIL import Image -from transformers import BatchFeature +from transformers import AutoProcessor, BatchFeature from vllm.model_executor.models.lightonocr import LightOnOCRProcessingInfo from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo @@ -22,25 +22,41 @@ _MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" _LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025" -def _processed_pixel_values( +def _process_images_with_hf( hf_processor, images: list[Image.Image], mm_processor_kwargs: dict[str, object], -) -> list[torch.Tensor]: - """Resize via the HF image processor and un-pad to per-image H×W.""" - image_processor = hf_processor.image_processor - hf_out = image_processor(images=images, return_tensors="pt", **mm_processor_kwargs) +) -> tuple[list[torch.Tensor], list[int]]: + """Process images and placeholders through the public HF processor.""" + hf_out = hf_processor( + text=hf_processor.image_token * len(images), + images=images, + return_tensors="pt", + **mm_processor_kwargs, + ) pixel_values = hf_out["pixel_values"] image_sizes = hf_out["image_sizes"] - return [p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)] + unpadded = [p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)] + + special_token_ids = { + hf_processor.image_token_id, + hf_processor.image_break_token_id, + hf_processor.image_end_token_id, + } + placeholder_tokens = [ + token_id + for token_id in hf_out["input_ids"][0].tolist() + if token_id in special_token_ids + ] + return unpadded, placeholder_tokens -def _placeholder_count_from_prompt_updates( +def _placeholder_tokens_from_prompt_updates( processor, images: list[Image.Image], pixel_values: list[torch.Tensor], mm_processor_kwargs: dict[str, object], -) -> int: +) -> list[int]: hf_inputs = BatchFeature({"pixel_values": pixel_values}) fields_config = processor._get_mm_fields_config(hf_inputs, mm_processor_kwargs) out_mm_kwargs = MultiModalKwargsItems.from_hf_inputs(hf_inputs, fields_config) @@ -49,13 +65,11 @@ def _placeholder_count_from_prompt_updates( updates = processor._get_prompt_updates( mm_items, mm_processor_kwargs, out_mm_kwargs ) - image_token_id = processor.info.get_hf_config().image_token_index - - total = 0 + placeholder_tokens: list[int] = [] for item_idx in range(len(images)): details = updates[0].resolve(item_idx).content - total += details.full.count(image_token_id) - return total + placeholder_tokens.extend(details.full) + return placeholder_tokens def _expected_placeholder_tokens_per_image( @@ -72,12 +86,12 @@ def _expected_placeholder_tokens_per_image( patch_h = patch_w = int(patch_size) spatial_merge_size = getattr(hf_processor, "spatial_merge_size", 1) - assert image_h % patch_h == 0 - assert image_w % patch_w == 0 + merged_patch_h = patch_h * spatial_merge_size + merged_patch_w = patch_w * spatial_merge_size + assert image_h % merged_patch_h == 0 + assert image_w % merged_patch_w == 0 - return (image_h // (patch_h * spatial_merge_size)) * ( - image_w // (patch_w * spatial_merge_size) - ) + return (image_h // merged_patch_h) * (image_w // merged_patch_w) @pytest.mark.parametrize("model_id", [_MODEL_ID]) @@ -87,8 +101,8 @@ def _expected_placeholder_tokens_per_image( ({}, (448, 448), 256), ({"size": {"longest_edge": 1008}}, (1540, 1540), 1296), ({"size": {"longest_edge": 1288}}, (1536, 1187), 1656), - ({"size": {"longest_edge": 1008}}, (29, 29), 1), - ({"size": {"longest_edge": 1000}}, (1540, 1700), 1152), + ({"size": {"longest_edge": 1008}}, (29, 29), 4), + ({"size": {"longest_edge": 1000}}, (1540, 1700), 1188), ], ) @pytest.mark.parametrize("num_imgs", [1, 2]) @@ -109,21 +123,29 @@ def test_processor_size_override( ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs - hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) + hf_processor = AutoProcessor.from_pretrained( + model_id, + fix_mistral_regex=True, + ) dummy_image = Image.new("RGB", image_size, color=(127, 127, 127)) images = [dummy_image] * num_imgs merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) - pixel_values = _processed_pixel_values(hf_processor, images, merged_mm_kwargs) + pixel_values, hf_placeholder_tokens = _process_images_with_hf( + hf_processor, images, merged_mm_kwargs + ) - image_token_count = _placeholder_count_from_prompt_updates( + prompt_update_tokens = _placeholder_tokens_from_prompt_updates( processor, images, pixel_values, hf_processor_mm_kwargs ) expected_from_pixel_values = _expected_placeholder_tokens_per_image( hf_processor, pixel_values[0] ) assert expected_from_pixel_values == expected_toks_per_img - assert image_token_count == expected_from_pixel_values * num_imgs + assert hf_placeholder_tokens.count(hf_processor.image_token_id) == ( + expected_from_pixel_values * num_imgs + ) + assert prompt_update_tokens == hf_placeholder_tokens def test_lightonocr_keeps_vision_config_image_size(): diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index 41352dc427..a70d8e1de5 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -192,18 +192,13 @@ class Mistral3HFEncoderInfo(PixtralHFEncoderInfo): image_width = math.floor(image_width / ratio) image_height = math.floor(image_height / ratio) - patch_size = self.vision_config.patch_size assert isinstance(self.hf_config, Mistral3Config) - spatial_merge_size = self.hf_config.spatial_merge_size + merged_patch_size = ( + self.vision_config.patch_size * self.hf_config.spatial_merge_size + ) - # The HF processor rounds each dimension up to the vision patch size - # before the projector drops incomplete spatial-merge groups. This is - # not equivalent to rounding directly to the merged patch size. - num_width_patches = (image_width - 1) // patch_size + 1 - num_height_patches = (image_height - 1) // patch_size + 1 - - ncols = num_width_patches // spatial_merge_size - nrows = num_height_patches // spatial_merge_size + ncols = (image_width - 1) // merged_patch_size + 1 + nrows = (image_height - 1) // merged_patch_size + 1 return ncols, nrows