[Model] Support precomputed image embeddings for LightOnOCR (#61585)
Signed-off-by: Rapisurazurite <43494437+Rapisurazurite@users.noreply.github.com>
This commit is contained in:
@@ -4,15 +4,14 @@ from collections.abc import Iterable, Mapping, Sequence
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
PixtralVisionConfig,
|
||||
)
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.models.mistral3 import (
|
||||
Mistral3DummyInputsBuilder,
|
||||
Mistral3ForConditionalGeneration,
|
||||
Mistral3ImageEmbeddingInputs,
|
||||
Mistral3ImageInputs,
|
||||
Mistral3MultiModalProjector,
|
||||
Mistral3ProcessingInfo,
|
||||
init_vision_tower_for_mistral3,
|
||||
@@ -26,12 +25,15 @@ from vllm.model_executor.models.utils import (
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
|
||||
from vllm.multimodal.parse import (
|
||||
ImageEmbeddingItems,
|
||||
ImageProcessorItems,
|
||||
MultiModalDataItems,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseMultiModalProcessor,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
|
||||
|
||||
@@ -102,18 +104,20 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
|
||||
hf_config = self.info.get_hf_config()
|
||||
image_token_id = hf_config.image_token_index
|
||||
|
||||
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
|
||||
encoder_info = PixtralHFEncoderInfo(hf_config)
|
||||
|
||||
def replace(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
size = images.get_image_size(item_idx)
|
||||
ncols, nrows = encoder_info.get_patch_grid_size(
|
||||
image_width=size.width, image_height=size.height
|
||||
images = mm_items.get_items(
|
||||
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
||||
)
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
num_image_tokens = images.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(images, ImageProcessorItems)
|
||||
size = images.get_image_size(item_idx)
|
||||
num_image_tokens = self.info.get_num_image_tokens(
|
||||
image_width=size.width, image_height=size.height
|
||||
)
|
||||
# break/end tokens are not used in LightOnOCR
|
||||
tokens = [image_token_id] * (ncols * nrows)
|
||||
return PromptUpdateDetails.select_token_id(tokens, image_token_id)
|
||||
return [image_token_id] * num_image_tokens
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
@@ -178,6 +182,15 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> Mistral3ImageInputs | None:
|
||||
image_embeds = kwargs.get("image_embeds")
|
||||
if image_embeds is not None:
|
||||
return Mistral3ImageEmbeddingInputs(data=image_embeds)
|
||||
|
||||
return super()._parse_and_validate_image_input(**kwargs)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal, cast
|
||||
from typing import Annotated, Any, Literal, TypeAlias, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -76,6 +76,19 @@ class Mistral3ImagePixelInputs(TensorSchema):
|
||||
]
|
||||
|
||||
|
||||
class Mistral3ImageEmbeddingInputs(TensorSchema):
|
||||
"""Projected features with shape (num_images, feature_size, hidden_size)."""
|
||||
|
||||
type: Literal["image_embeds"] = "image_embeds"
|
||||
data: Annotated[
|
||||
torch.Tensor | list[torch.Tensor],
|
||||
TensorShape("bn", "ifs", "hs", dynamic_dims={"ifs"}),
|
||||
]
|
||||
|
||||
|
||||
Mistral3ImageInputs: TypeAlias = Mistral3ImagePixelInputs | Mistral3ImageEmbeddingInputs
|
||||
|
||||
|
||||
class Mistral3PatchMerger(nn.Module):
|
||||
"""Learned merging of spatial_merge_size ** 2 patches."""
|
||||
|
||||
@@ -494,7 +507,7 @@ class Mistral3ForConditionalGeneration(
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> Mistral3ImagePixelInputs | None:
|
||||
) -> Mistral3ImageInputs | None:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
image_embeds = kwargs.pop("image_embeds", None)
|
||||
|
||||
@@ -508,8 +521,8 @@ class Mistral3ForConditionalGeneration(
|
||||
|
||||
def _process_image_input(
|
||||
self,
|
||||
image_input: Mistral3ImagePixelInputs,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
image_input: Mistral3ImageInputs,
|
||||
) -> MultiModalEmbeddings:
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["data"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user