[Model] Support precomputed image embeddings for LightOnOCR (#61585)
Signed-off-by: Rapisurazurite <43494437+Rapisurazurite@users.noreply.github.com>
This commit is contained in:
@@ -581,7 +581,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
|||||||
| `KimiK25ForConditionalGeneration` | Kimi-K2.5 | T + I<sup>+</sup> | `moonshotai/Kimi-K2.5` | | ✅︎ |
|
| `KimiK25ForConditionalGeneration` | Kimi-K2.5 | T + I<sup>+</sup> | `moonshotai/Kimi-K2.5` | | ✅︎ |
|
||||||
| `KimiK3ForConditionalGeneration` | Kimi-K3 | T + I<sup>+</sup> | `moonshotai/Kimi-K3` | | ✅︎ |
|
| `KimiK3ForConditionalGeneration` | Kimi-K3 | T + I<sup>+</sup> | `moonshotai/Kimi-K3` | | ✅︎ |
|
||||||
| `KimiVLForConditionalGeneration` | Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking | T + I<sup>+</sup> | `moonshotai/Kimi-VL-A3B-Instruct`, `moonshotai/Kimi-VL-A3B-Thinking` | | ✅︎ |
|
| `KimiVLForConditionalGeneration` | Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking | T + I<sup>+</sup> | `moonshotai/Kimi-VL-A3B-Instruct`, `moonshotai/Kimi-VL-A3B-Thinking` | | ✅︎ |
|
||||||
| `LightOnOCRForConditionalGeneration` | LightOnOCR-1B | T + I<sup>+</sup> | `lightonai/LightOnOCR-1B`, etc | ✅︎ | ✅︎ |
|
| `LightOnOCRForConditionalGeneration` | LightOnOCR-1B | T + I<sup>E+</sup> | `lightonai/LightOnOCR-1B`, etc | ✅︎ | ✅︎ |
|
||||||
| `Lfm2VlForConditionalGeneration` | LFM2-VL | T + I<sup>+</sup> | `LiquidAI/LFM2-VL-450M`, `LiquidAI/LFM2-VL-3B`, `LiquidAI/LFM2-VL-8B-A1B`, etc. | ✅︎ | ✅︎ |
|
| `Lfm2VlForConditionalGeneration` | LFM2-VL | T + I<sup>+</sup> | `LiquidAI/LFM2-VL-450M`, `LiquidAI/LFM2-VL-3B`, `LiquidAI/LFM2-VL-8B-A1B`, etc. | ✅︎ | ✅︎ |
|
||||||
| `Llama4ForConditionalGeneration` | Llama 4 | T + I<sup>+</sup> | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
|
| `Llama4ForConditionalGeneration` | Llama 4 | T + I<sup>+</sup> | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
|
||||||
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
|
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Tests for LightOnOCR precomputed image embeddings."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
|
||||||
|
from ...utils import build_model_context
|
||||||
|
|
||||||
|
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
|
||||||
|
_MODEL_CONFIG_KWARGS = {"config_format": "hf"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.usefixtures("default_vllm_config")
|
||||||
|
@pytest.mark.parametrize("tensor_input", [False, True])
|
||||||
|
@pytest.mark.parametrize("prompt_as_tokens", [False, True])
|
||||||
|
def test_lightonocr_image_embeddings_preserve_lengths_and_values(
|
||||||
|
tensor_input: bool, prompt_as_tokens: bool
|
||||||
|
):
|
||||||
|
"""Reserve each image's feature count and pass projected embeddings through."""
|
||||||
|
ctx = build_model_context(
|
||||||
|
_LIGHTON_MODEL_ID,
|
||||||
|
limit_mm_per_prompt={"image": 2},
|
||||||
|
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||||
|
)
|
||||||
|
ctx.model_config.get_multimodal_config().enable_mm_embeds = True
|
||||||
|
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||||
|
hf_processor = processor.info.get_hf_processor()
|
||||||
|
hidden_size = ctx.model_config.get_hidden_size()
|
||||||
|
feature_sizes = [3, 3] if tensor_input else [3, 7]
|
||||||
|
embeddings = [
|
||||||
|
torch.full((size, hidden_size), float(i))
|
||||||
|
for i, size in enumerate(feature_sizes)
|
||||||
|
]
|
||||||
|
image_input = torch.stack(embeddings) if tensor_input else embeddings
|
||||||
|
prompt = " ".join([hf_processor.image_token] * len(embeddings))
|
||||||
|
if prompt_as_tokens:
|
||||||
|
prompt = ctx.get_tokenizer().encode(prompt)
|
||||||
|
|
||||||
|
items = processor.info.parse_mm_data({"image": image_input})
|
||||||
|
processed = processor(prompt, mm_items=items)
|
||||||
|
|
||||||
|
assert processed["prompt_token_ids"].count(hf_processor.image_token_id) == sum(
|
||||||
|
feature_sizes
|
||||||
|
)
|
||||||
|
assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes
|
||||||
|
for item, expected in zip(processed["mm_kwargs"]["image"], embeddings):
|
||||||
|
torch.testing.assert_close(item.get_data()["image_embeds"], expected)
|
||||||
@@ -4,15 +4,14 @@ from collections.abc import Iterable, Mapping, Sequence
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from transformers import (
|
from transformers import BatchFeature
|
||||||
BatchFeature,
|
|
||||||
PixtralVisionConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.model_executor.models.mistral3 import (
|
from vllm.model_executor.models.mistral3 import (
|
||||||
Mistral3DummyInputsBuilder,
|
Mistral3DummyInputsBuilder,
|
||||||
Mistral3ForConditionalGeneration,
|
Mistral3ForConditionalGeneration,
|
||||||
|
Mistral3ImageEmbeddingInputs,
|
||||||
|
Mistral3ImageInputs,
|
||||||
Mistral3MultiModalProjector,
|
Mistral3MultiModalProjector,
|
||||||
Mistral3ProcessingInfo,
|
Mistral3ProcessingInfo,
|
||||||
init_vision_tower_for_mistral3,
|
init_vision_tower_for_mistral3,
|
||||||
@@ -26,12 +25,15 @@ from vllm.model_executor.models.utils import (
|
|||||||
)
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||||
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
|
from vllm.multimodal.parse import (
|
||||||
|
ImageEmbeddingItems,
|
||||||
|
ImageProcessorItems,
|
||||||
|
MultiModalDataItems,
|
||||||
|
)
|
||||||
from vllm.multimodal.processing import (
|
from vllm.multimodal.processing import (
|
||||||
BaseMultiModalProcessor,
|
BaseMultiModalProcessor,
|
||||||
PromptReplacement,
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
PromptUpdateDetails,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -102,18 +104,20 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
|
|||||||
hf_config = self.info.get_hf_config()
|
hf_config = self.info.get_hf_config()
|
||||||
image_token_id = hf_config.image_token_index
|
image_token_id = hf_config.image_token_index
|
||||||
|
|
||||||
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
|
|
||||||
encoder_info = PixtralHFEncoderInfo(hf_config)
|
|
||||||
|
|
||||||
def replace(item_idx: int):
|
def replace(item_idx: int):
|
||||||
images = mm_items.get_items("image", ImageProcessorItems)
|
images = mm_items.get_items(
|
||||||
size = images.get_image_size(item_idx)
|
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
||||||
ncols, nrows = encoder_info.get_patch_grid_size(
|
|
||||||
image_width=size.width, image_height=size.height
|
|
||||||
)
|
)
|
||||||
|
if isinstance(images, ImageEmbeddingItems):
|
||||||
|
num_image_tokens = images.get_feature_size(item_idx)
|
||||||
|
else:
|
||||||
|
assert isinstance(images, ImageProcessorItems)
|
||||||
|
size = images.get_image_size(item_idx)
|
||||||
|
num_image_tokens = self.info.get_num_image_tokens(
|
||||||
|
image_width=size.width, image_height=size.height
|
||||||
|
)
|
||||||
# break/end tokens are not used in LightOnOCR
|
# break/end tokens are not used in LightOnOCR
|
||||||
tokens = [image_token_id] * (ncols * nrows)
|
return [image_token_id] * num_image_tokens
|
||||||
return PromptUpdateDetails.select_token_id(tokens, image_token_id)
|
|
||||||
|
|
||||||
return [
|
return [
|
||||||
PromptReplacement(
|
PromptReplacement(
|
||||||
@@ -178,6 +182,15 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
|
|||||||
self.language_model.make_empty_intermediate_tensors
|
self.language_model.make_empty_intermediate_tensors
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _parse_and_validate_image_input(
|
||||||
|
self, **kwargs: object
|
||||||
|
) -> Mistral3ImageInputs | None:
|
||||||
|
image_embeds = kwargs.get("image_embeds")
|
||||||
|
if image_embeds is not None:
|
||||||
|
return Mistral3ImageEmbeddingInputs(data=image_embeds)
|
||||||
|
|
||||||
|
return super()._parse_and_validate_image_input(**kwargs)
|
||||||
|
|
||||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||||
loader = AutoWeightsLoader(self)
|
loader = AutoWeightsLoader(self)
|
||||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
import math
|
import math
|
||||||
from collections.abc import Iterable, Mapping, Sequence
|
from collections.abc import Iterable, Mapping, Sequence
|
||||||
from typing import Annotated, Any, Literal, cast
|
from typing import Annotated, Any, Literal, TypeAlias, cast
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
@@ -76,6 +76,19 @@ class Mistral3ImagePixelInputs(TensorSchema):
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class Mistral3ImageEmbeddingInputs(TensorSchema):
|
||||||
|
"""Projected features with shape (num_images, feature_size, hidden_size)."""
|
||||||
|
|
||||||
|
type: Literal["image_embeds"] = "image_embeds"
|
||||||
|
data: Annotated[
|
||||||
|
torch.Tensor | list[torch.Tensor],
|
||||||
|
TensorShape("bn", "ifs", "hs", dynamic_dims={"ifs"}),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
Mistral3ImageInputs: TypeAlias = Mistral3ImagePixelInputs | Mistral3ImageEmbeddingInputs
|
||||||
|
|
||||||
|
|
||||||
class Mistral3PatchMerger(nn.Module):
|
class Mistral3PatchMerger(nn.Module):
|
||||||
"""Learned merging of spatial_merge_size ** 2 patches."""
|
"""Learned merging of spatial_merge_size ** 2 patches."""
|
||||||
|
|
||||||
@@ -494,7 +507,7 @@ class Mistral3ForConditionalGeneration(
|
|||||||
|
|
||||||
def _parse_and_validate_image_input(
|
def _parse_and_validate_image_input(
|
||||||
self, **kwargs: object
|
self, **kwargs: object
|
||||||
) -> Mistral3ImagePixelInputs | None:
|
) -> Mistral3ImageInputs | None:
|
||||||
pixel_values = kwargs.pop("pixel_values", None)
|
pixel_values = kwargs.pop("pixel_values", None)
|
||||||
image_embeds = kwargs.pop("image_embeds", None)
|
image_embeds = kwargs.pop("image_embeds", None)
|
||||||
|
|
||||||
@@ -508,8 +521,8 @@ class Mistral3ForConditionalGeneration(
|
|||||||
|
|
||||||
def _process_image_input(
|
def _process_image_input(
|
||||||
self,
|
self,
|
||||||
image_input: Mistral3ImagePixelInputs,
|
image_input: Mistral3ImageInputs,
|
||||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
) -> MultiModalEmbeddings:
|
||||||
if image_input["type"] == "image_embeds":
|
if image_input["type"] == "image_embeds":
|
||||||
return image_input["data"]
|
return image_input["data"]
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user