[Bugfix][Model] Mistral3: align placeholder grid with public processor (#53758)
Signed-off-by: Oliver Holworthy <1216955+oliverholworthy@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
59f1527280
commit
9598487bc1
@@ -5,7 +5,7 @@
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature
|
||||
from transformers import AutoProcessor, BatchFeature
|
||||
|
||||
from vllm.model_executor.models.lightonocr import LightOnOCRProcessingInfo
|
||||
from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo
|
||||
@@ -22,25 +22,41 @@ _MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
|
||||
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
|
||||
|
||||
|
||||
def _processed_pixel_values(
|
||||
def _process_images_with_hf(
|
||||
hf_processor,
|
||||
images: list[Image.Image],
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
) -> list[torch.Tensor]:
|
||||
"""Resize via the HF image processor and un-pad to per-image H×W."""
|
||||
image_processor = hf_processor.image_processor
|
||||
hf_out = image_processor(images=images, return_tensors="pt", **mm_processor_kwargs)
|
||||
) -> tuple[list[torch.Tensor], list[int]]:
|
||||
"""Process images and placeholders through the public HF processor."""
|
||||
hf_out = hf_processor(
|
||||
text=hf_processor.image_token * len(images),
|
||||
images=images,
|
||||
return_tensors="pt",
|
||||
**mm_processor_kwargs,
|
||||
)
|
||||
pixel_values = hf_out["pixel_values"]
|
||||
image_sizes = hf_out["image_sizes"]
|
||||
return [p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)]
|
||||
unpadded = [p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)]
|
||||
|
||||
special_token_ids = {
|
||||
hf_processor.image_token_id,
|
||||
hf_processor.image_break_token_id,
|
||||
hf_processor.image_end_token_id,
|
||||
}
|
||||
placeholder_tokens = [
|
||||
token_id
|
||||
for token_id in hf_out["input_ids"][0].tolist()
|
||||
if token_id in special_token_ids
|
||||
]
|
||||
return unpadded, placeholder_tokens
|
||||
|
||||
|
||||
def _placeholder_count_from_prompt_updates(
|
||||
def _placeholder_tokens_from_prompt_updates(
|
||||
processor,
|
||||
images: list[Image.Image],
|
||||
pixel_values: list[torch.Tensor],
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
) -> int:
|
||||
) -> list[int]:
|
||||
hf_inputs = BatchFeature({"pixel_values": pixel_values})
|
||||
fields_config = processor._get_mm_fields_config(hf_inputs, mm_processor_kwargs)
|
||||
out_mm_kwargs = MultiModalKwargsItems.from_hf_inputs(hf_inputs, fields_config)
|
||||
@@ -49,13 +65,11 @@ def _placeholder_count_from_prompt_updates(
|
||||
updates = processor._get_prompt_updates(
|
||||
mm_items, mm_processor_kwargs, out_mm_kwargs
|
||||
)
|
||||
image_token_id = processor.info.get_hf_config().image_token_index
|
||||
|
||||
total = 0
|
||||
placeholder_tokens: list[int] = []
|
||||
for item_idx in range(len(images)):
|
||||
details = updates[0].resolve(item_idx).content
|
||||
total += details.full.count(image_token_id)
|
||||
return total
|
||||
placeholder_tokens.extend(details.full)
|
||||
return placeholder_tokens
|
||||
|
||||
|
||||
def _expected_placeholder_tokens_per_image(
|
||||
@@ -72,12 +86,12 @@ def _expected_placeholder_tokens_per_image(
|
||||
patch_h = patch_w = int(patch_size)
|
||||
|
||||
spatial_merge_size = getattr(hf_processor, "spatial_merge_size", 1)
|
||||
assert image_h % patch_h == 0
|
||||
assert image_w % patch_w == 0
|
||||
merged_patch_h = patch_h * spatial_merge_size
|
||||
merged_patch_w = patch_w * spatial_merge_size
|
||||
assert image_h % merged_patch_h == 0
|
||||
assert image_w % merged_patch_w == 0
|
||||
|
||||
return (image_h // (patch_h * spatial_merge_size)) * (
|
||||
image_w // (patch_w * spatial_merge_size)
|
||||
)
|
||||
return (image_h // merged_patch_h) * (image_w // merged_patch_w)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [_MODEL_ID])
|
||||
@@ -87,8 +101,8 @@ def _expected_placeholder_tokens_per_image(
|
||||
({}, (448, 448), 256),
|
||||
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
|
||||
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
|
||||
({"size": {"longest_edge": 1008}}, (29, 29), 1),
|
||||
({"size": {"longest_edge": 1000}}, (1540, 1700), 1152),
|
||||
({"size": {"longest_edge": 1008}}, (29, 29), 4),
|
||||
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@@ -109,21 +123,29 @@ def test_processor_size_override(
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
hf_processor = AutoProcessor.from_pretrained(
|
||||
model_id,
|
||||
fix_mistral_regex=True,
|
||||
)
|
||||
|
||||
dummy_image = Image.new("RGB", image_size, color=(127, 127, 127))
|
||||
images = [dummy_image] * num_imgs
|
||||
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
pixel_values = _processed_pixel_values(hf_processor, images, merged_mm_kwargs)
|
||||
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
|
||||
hf_processor, images, merged_mm_kwargs
|
||||
)
|
||||
|
||||
image_token_count = _placeholder_count_from_prompt_updates(
|
||||
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
|
||||
processor, images, pixel_values, hf_processor_mm_kwargs
|
||||
)
|
||||
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
|
||||
hf_processor, pixel_values[0]
|
||||
)
|
||||
assert expected_from_pixel_values == expected_toks_per_img
|
||||
assert image_token_count == expected_from_pixel_values * num_imgs
|
||||
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == (
|
||||
expected_from_pixel_values * num_imgs
|
||||
)
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
def test_lightonocr_keeps_vision_config_image_size():
|
||||
|
||||
@@ -192,18 +192,13 @@ class Mistral3HFEncoderInfo(PixtralHFEncoderInfo):
|
||||
image_width = math.floor(image_width / ratio)
|
||||
image_height = math.floor(image_height / ratio)
|
||||
|
||||
patch_size = self.vision_config.patch_size
|
||||
assert isinstance(self.hf_config, Mistral3Config)
|
||||
spatial_merge_size = self.hf_config.spatial_merge_size
|
||||
merged_patch_size = (
|
||||
self.vision_config.patch_size * self.hf_config.spatial_merge_size
|
||||
)
|
||||
|
||||
# The HF processor rounds each dimension up to the vision patch size
|
||||
# before the projector drops incomplete spatial-merge groups. This is
|
||||
# not equivalent to rounding directly to the merged patch size.
|
||||
num_width_patches = (image_width - 1) // patch_size + 1
|
||||
num_height_patches = (image_height - 1) // patch_size + 1
|
||||
|
||||
ncols = num_width_patches // spatial_merge_size
|
||||
nrows = num_height_patches // spatial_merge_size
|
||||
ncols = (image_width - 1) // merged_patch_size + 1
|
||||
nrows = (image_height - 1) // merged_patch_size + 1
|
||||
return ncols, nrows
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user