[Bugfix][Model] Mistral3: fix image placeholder grid for processor size overrides (#52874)

Signed-off-by: Anmol Gupta <14880251+anmolgupt@users.noreply.github.com>
This commit is contained in:
anmolgupt
2026-08-22 21:34:38 -07:00
committed by GitHub
parent d569e71bd0
commit a3561ef8e4
3 changed files with 210 additions and 7 deletions
+13 -2
View File
@@ -38,7 +38,18 @@ from vllm.multimodal.processing import (
_I = TypeVar("_I", bound=Mistral3ProcessingInfo)
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
def get_vision_encoder_info(
self,
mm_processor_kwargs: Mapping[str, object] | None = None,
) -> PixtralHFEncoderInfo:
# LightOnOCR prompt updates use vision_config.image_size directly;
# preserve that behavior rather than inheriting Mistral3's processor
# longest_edge override.
return PixtralHFEncoderInfo(self.get_hf_config())
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
@@ -124,7 +135,7 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
@MULTIMODAL_REGISTRY.register_processor(
LightOnOCRMultiModalProcessor,
info=Mistral3ProcessingInfo,
info=LightOnOCRProcessingInfo,
dummy_inputs=Mistral3DummyInputsBuilder,
)
class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
+54 -5
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal
@@ -22,7 +23,11 @@ from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
from vllm.multimodal.parse import (
ImageProcessorItems,
ImageSize,
MultiModalDataItems,
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
@@ -50,7 +55,6 @@ from .utils import (
init_vllm_registered_model,
maybe_prefix,
)
from .vision import get_vision_encoder_info
class Mistral3ImagePixelInputs(TensorSchema):
@@ -170,12 +174,57 @@ class Mistral3MultiModalProjector(nn.Module):
return hidden_states
class Mistral3HFEncoderInfo(PixtralHFEncoderInfo):
def __init__(self, hf_config: Mistral3Config, image_size: int) -> None:
super().__init__(hf_config)
self.image_size = image_size
def get_image_size(self) -> int:
return self.image_size
def get_patch_grid_size(
self,
*,
image_width: int,
image_height: int,
) -> tuple[int, int]:
max_size = self.get_image_size()
ratio = max(image_width / max_size, image_height / max_size)
if ratio > 1:
image_width = math.floor(image_width / ratio)
image_height = math.floor(image_height / ratio)
patch_size = self.vision_config.patch_size
spatial_merge_size = self.hf_config.spatial_merge_size
# The HF processor rounds each dimension up to the vision patch size
# before the projector drops incomplete spatial-merge groups. This is
# not equivalent to rounding directly to the merged patch size.
num_width_patches = (image_width - 1) // patch_size + 1
num_height_patches = (image_height - 1) // patch_size + 1
ncols = num_width_patches // spatial_merge_size
nrows = num_height_patches // spatial_merge_size
return ncols, nrows
class Mistral3ProcessingInfo(BaseProcessingInfo):
def get_hf_config(self) -> Mistral3Config:
return self.ctx.get_hf_config(Mistral3Config)
def get_vision_encoder_info(self):
return get_vision_encoder_info(self.get_hf_config())
def get_vision_encoder_info(
self,
mm_processor_kwargs: Mapping[str, object] | None = None,
) -> Mistral3HFEncoderInfo:
processor = self.get_hf_processor()
size = processor.image_processor.size
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
if override_size := merged_kwargs.get("size"):
size = size | override_size
image_size = size["longest_edge"]
return Mistral3HFEncoderInfo(self.get_hf_config(), image_size)
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
@@ -284,7 +333,7 @@ class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo
image_end_id = vocab[processor.image_end_token]
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
encoder_info = PixtralHFEncoderInfo(hf_config)
encoder_info = self.info.get_vision_encoder_info(hf_processor_mm_kwargs)
def get_replacement(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)