[Bugfix][Model] Mistral3: fix image placeholder grid for processor size overrides (#52874)
Signed-off-by: Anmol Gupta <14880251+anmolgupt@users.noreply.github.com>
This commit is contained in:
@@ -38,7 +38,18 @@ from vllm.multimodal.processing import (
|
||||
_I = TypeVar("_I", bound=Mistral3ProcessingInfo)
|
||||
|
||||
|
||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
|
||||
def get_vision_encoder_info(
|
||||
self,
|
||||
mm_processor_kwargs: Mapping[str, object] | None = None,
|
||||
) -> PixtralHFEncoderInfo:
|
||||
# LightOnOCR prompt updates use vision_config.image_size directly;
|
||||
# preserve that behavior rather than inheriting Mistral3's processor
|
||||
# longest_edge override.
|
||||
return PixtralHFEncoderInfo(self.get_hf_config())
|
||||
|
||||
|
||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
@@ -124,7 +135,7 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
LightOnOCRMultiModalProcessor,
|
||||
info=Mistral3ProcessingInfo,
|
||||
info=LightOnOCRProcessingInfo,
|
||||
dummy_inputs=Mistral3DummyInputsBuilder,
|
||||
)
|
||||
class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Literal
|
||||
|
||||
@@ -22,7 +23,11 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
|
||||
from vllm.multimodal.parse import (
|
||||
ImageProcessorItems,
|
||||
ImageSize,
|
||||
MultiModalDataItems,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
@@ -50,7 +55,6 @@ from .utils import (
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import get_vision_encoder_info
|
||||
|
||||
|
||||
class Mistral3ImagePixelInputs(TensorSchema):
|
||||
@@ -170,12 +174,57 @@ class Mistral3MultiModalProjector(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Mistral3HFEncoderInfo(PixtralHFEncoderInfo):
|
||||
def __init__(self, hf_config: Mistral3Config, image_size: int) -> None:
|
||||
super().__init__(hf_config)
|
||||
self.image_size = image_size
|
||||
|
||||
def get_image_size(self) -> int:
|
||||
return self.image_size
|
||||
|
||||
def get_patch_grid_size(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
) -> tuple[int, int]:
|
||||
max_size = self.get_image_size()
|
||||
ratio = max(image_width / max_size, image_height / max_size)
|
||||
|
||||
if ratio > 1:
|
||||
image_width = math.floor(image_width / ratio)
|
||||
image_height = math.floor(image_height / ratio)
|
||||
|
||||
patch_size = self.vision_config.patch_size
|
||||
spatial_merge_size = self.hf_config.spatial_merge_size
|
||||
|
||||
# The HF processor rounds each dimension up to the vision patch size
|
||||
# before the projector drops incomplete spatial-merge groups. This is
|
||||
# not equivalent to rounding directly to the merged patch size.
|
||||
num_width_patches = (image_width - 1) // patch_size + 1
|
||||
num_height_patches = (image_height - 1) // patch_size + 1
|
||||
|
||||
ncols = num_width_patches // spatial_merge_size
|
||||
nrows = num_height_patches // spatial_merge_size
|
||||
return ncols, nrows
|
||||
|
||||
|
||||
class Mistral3ProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_config(self) -> Mistral3Config:
|
||||
return self.ctx.get_hf_config(Mistral3Config)
|
||||
|
||||
def get_vision_encoder_info(self):
|
||||
return get_vision_encoder_info(self.get_hf_config())
|
||||
def get_vision_encoder_info(
|
||||
self,
|
||||
mm_processor_kwargs: Mapping[str, object] | None = None,
|
||||
) -> Mistral3HFEncoderInfo:
|
||||
processor = self.get_hf_processor()
|
||||
size = processor.image_processor.size
|
||||
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
|
||||
if override_size := merged_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
|
||||
image_size = size["longest_edge"]
|
||||
return Mistral3HFEncoderInfo(self.get_hf_config(), image_size)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
|
||||
@@ -284,7 +333,7 @@ class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo
|
||||
image_end_id = vocab[processor.image_end_token]
|
||||
|
||||
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
|
||||
encoder_info = PixtralHFEncoderInfo(hf_config)
|
||||
encoder_info = self.info.get_vision_encoder_info(hf_processor_mm_kwargs)
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
|
||||
Reference in New Issue
Block a user