[Core] Cast multimodal input in hf processor (#18862)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
This commit is contained in:
+24
-2
@@ -4,9 +4,12 @@ from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
|
||||
|
||||
import torch
|
||||
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.jsontree import JSONTree, json_map_leaves
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import resolve_mm_processor_kwargs
|
||||
@@ -21,6 +24,8 @@ _T = TypeVar("_T")
|
||||
_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
|
||||
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InputContext:
|
||||
@@ -134,7 +139,7 @@ class InputProcessingContext(InputContext):
|
||||
hf_processor: ProcessorMixin,
|
||||
data: Mapping[str, object],
|
||||
kwargs: Mapping[str, object] = {},
|
||||
) -> BatchFeature:
|
||||
) -> Union[BatchFeature, JSONTree]:
|
||||
"""
|
||||
Call `hf_processor` on the prompt `data`
|
||||
(text, image, audio...) with configurable options `kwargs`.
|
||||
@@ -154,8 +159,25 @@ class InputProcessingContext(InputContext):
|
||||
allow_var_kwargs=True,
|
||||
)
|
||||
|
||||
def maybe_cast_dtype(x):
|
||||
# This mimics the behavior of transformers.BatchFeature
|
||||
if isinstance(x, torch.Tensor) and x.is_floating_point():
|
||||
return x.to(dtype=self.model_config.dtype)
|
||||
return x
|
||||
|
||||
try:
|
||||
return hf_processor(**data, **merged_kwargs, return_tensors="pt")
|
||||
output = hf_processor(**data, **merged_kwargs, return_tensors="pt")
|
||||
# this emulates output.to(dtype=self.model_config.dtype)
|
||||
cast_output = json_map_leaves(maybe_cast_dtype, output)
|
||||
if isinstance(output, BatchFeature):
|
||||
return BatchFeature(cast_output)
|
||||
|
||||
logger.warning_once(
|
||||
f"{type(hf_processor).__name__} did not return `BatchFeature`. "
|
||||
"Make sure to match the behaviour of `ProcessorMixin` when "
|
||||
"implementing custom processors.")
|
||||
return cast_output
|
||||
|
||||
except Exception as exc:
|
||||
msg = (f"Failed to apply {type(hf_processor).__name__} "
|
||||
f"on data={data} with kwargs={merged_kwargs}")
|
||||
|
||||
Reference in New Issue
Block a user