[Model] Support precomputed image embeddings for LightOnOCR (#61585)

Signed-off-by: Rapisurazurite <43494437+Rapisurazurite@users.noreply.github.com>
This commit is contained in:
Lazurite
2026-10-14 10:54:09 +00:00
committed by GitHub
parent 81503550f3
commit 23fbc2ec2f
4 changed files with 96 additions and 20 deletions
+17 -4
View File
@@ -3,7 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, cast
from typing import Annotated, Any, Literal, TypeAlias, cast
import torch
import torch.nn as nn
@@ -76,6 +76,19 @@ class Mistral3ImagePixelInputs(TensorSchema):
]
class Mistral3ImageEmbeddingInputs(TensorSchema):
"""Projected features with shape (num_images, feature_size, hidden_size)."""
type: Literal["image_embeds"] = "image_embeds"
data: Annotated[
torch.Tensor | list[torch.Tensor],
TensorShape("bn", "ifs", "hs", dynamic_dims={"ifs"}),
]
Mistral3ImageInputs: TypeAlias = Mistral3ImagePixelInputs | Mistral3ImageEmbeddingInputs
class Mistral3PatchMerger(nn.Module):
"""Learned merging of spatial_merge_size ** 2 patches."""
@@ -494,7 +507,7 @@ class Mistral3ForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Mistral3ImagePixelInputs | None:
) -> Mistral3ImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -508,8 +521,8 @@ class Mistral3ForConditionalGeneration(
def _process_image_input(
self,
image_input: Mistral3ImagePixelInputs,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
image_input: Mistral3ImageInputs,
) -> MultiModalEmbeddings:
if image_input["type"] == "image_embeds":
return image_input["data"]