[Misc] Move processors to transformers_utils (#35953)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -4,7 +4,6 @@
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, fields
|
||||
from functools import cached_property
|
||||
from typing import Annotated, Literal
|
||||
|
||||
import torch
|
||||
@@ -13,10 +12,7 @@ import torch.nn.functional as F
|
||||
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
|
||||
from mistral_common.protocol.instruct.messages import UserMessage
|
||||
from mistral_common.protocol.instruct.request import ChatCompletionRequest
|
||||
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, PixtralVisionConfig, TensorType
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers import PixtralVisionConfig
|
||||
from transformers.models.pixtral.image_processing_pixtral import (
|
||||
_num_image_tokens as _get_pixtral_hf_num_image_tokens,
|
||||
)
|
||||
@@ -25,7 +21,6 @@ from transformers.models.pixtral.modeling_pixtral import (
|
||||
apply_rotary_pos_emb,
|
||||
position_ids_in_meshgrid,
|
||||
)
|
||||
from transformers.tokenization_utils_base import TextInput
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
@@ -66,6 +61,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.transformers_utils.processors.pixtral import MistralCommonPixtralProcessor
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (
|
||||
@@ -121,93 +117,6 @@ class PixtralImagePixelInputs(TensorSchema):
|
||||
]
|
||||
|
||||
|
||||
class PixtralProcessorAdapter:
|
||||
"""
|
||||
Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
|
||||
"""
|
||||
|
||||
def __init__(self, tokenizer: MistralTokenizer) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
@property
|
||||
def image_processor(self) -> ImageEncoder:
|
||||
image_encoder = self.tokenizer.instruct.mm_encoder
|
||||
assert isinstance(image_encoder, ImageEncoder)
|
||||
return image_encoder
|
||||
|
||||
@cached_property
|
||||
def image_break_id(self) -> int:
|
||||
return self.image_processor.special_ids.img_break
|
||||
|
||||
@cached_property
|
||||
def image_token_id(self) -> int:
|
||||
return self.image_processor.special_ids.img
|
||||
|
||||
@cached_property
|
||||
def image_end_id(self) -> int:
|
||||
return self.image_processor.special_ids.img_end
|
||||
|
||||
@cached_property
|
||||
def image_size(self) -> int:
|
||||
return self.image_processor.mm_config.max_image_size
|
||||
|
||||
@cached_property
|
||||
def patch_size(self) -> int:
|
||||
return self.image_processor.mm_config.image_patch_size
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: TextInput | list[TextInput] | None = None,
|
||||
images: ImageInput | list[ImageInput] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> Mapping[str, NestedTensors]:
|
||||
if text is None:
|
||||
text = []
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
if images is None:
|
||||
images = []
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
|
||||
if not images:
|
||||
input_ids = self.tokenizer(text).input_ids
|
||||
|
||||
return {"input_ids": torch.tensor(input_ids)}
|
||||
|
||||
# Allow dummy text, which is used for profiling as well as token inputs
|
||||
if any(len(t) > 0 for t in text):
|
||||
raise ValueError(
|
||||
"You've passed text inputs instead of token inputs. "
|
||||
"Make sure to process your input via `mistral_common`'s "
|
||||
"tokenizer or pass a chat completion request. "
|
||||
"For more info, see: "
|
||||
"https://github.com/vllm-project/vllm/issues/8411."
|
||||
)
|
||||
|
||||
images_processed = list[torch.Tensor]()
|
||||
images_tokens = list[torch.Tensor]()
|
||||
|
||||
for image in images:
|
||||
image_inputs = self.image_processor(ImageChunk(image=image))
|
||||
image_processed = torch.tensor(image_inputs.image)
|
||||
image_tokens = torch.tensor(image_inputs.tokens)
|
||||
|
||||
images_processed.append(image_processed)
|
||||
images_tokens.append(image_tokens)
|
||||
|
||||
return BatchFeature(
|
||||
{
|
||||
"input_ids": torch.cat(images_tokens)[None].expand(len(text), -1),
|
||||
"images": images_processed,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class PixtralProcessingInfo(BaseProcessingInfo):
|
||||
def get_tokenizer(self) -> MistralTokenizer:
|
||||
tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
|
||||
@@ -216,28 +125,19 @@ class PixtralProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
return tokenizer
|
||||
|
||||
def get_hf_processor(self) -> PixtralProcessorAdapter:
|
||||
return PixtralProcessorAdapter(self.get_tokenizer())
|
||||
def get_hf_processor(self, **kwargs) -> MistralCommonPixtralProcessor:
|
||||
return self.ctx.init_processor(
|
||||
MistralCommonPixtralProcessor,
|
||||
tokenizer=self.get_tokenizer(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None}
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
processor: PixtralProcessorAdapter,
|
||||
) -> int:
|
||||
ncols, nrows = processor.image_processor._image_to_num_tokens(
|
||||
Image.new("RGB", (image_width, image_height))
|
||||
)
|
||||
|
||||
return ncols * nrows
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
image_processor = self.get_hf_processor().image_processor
|
||||
max_image_size = image_processor.mm_config.max_image_size
|
||||
max_image_size = image_processor.mm_encoder.mm_config.max_image_size
|
||||
|
||||
return ImageSize(width=max_image_size, height=max_image_size)
|
||||
|
||||
@@ -321,8 +221,9 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_size = images.get_image_size(item_idx)
|
||||
|
||||
ncols, nrows = processor.image_processor._image_to_num_tokens(
|
||||
Image.new("RGB", (image_size.width, image_size.height))
|
||||
_, nrows, ncols = processor.image_processor.get_number_of_image_patches(
|
||||
image_size.height,
|
||||
image_size.width,
|
||||
)
|
||||
|
||||
tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
|
||||
|
||||
Reference in New Issue
Block a user