[Misc]] Move processing context to multimodal directory (#25548)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-09-24 08:15:00 +00:00
committed by GitHub
parent 27ec3c78f3
commit 6488f3481b
13 changed files with 262 additions and 242 deletions
+226 -5
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from abc import ABC, abstractmethod
from collections import defaultdict
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
@@ -7,18 +8,20 @@ from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
from dataclasses import dataclass, field, replace
from enum import Enum
from functools import lru_cache
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
TypeVar, Union, cast)
from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional,
Protocol, Union, cast, overload)
import regex as re
import torch
from typing_extensions import assert_never
from typing_extensions import TypeVar, assert_never
from vllm.inputs import InputProcessingContext
from vllm.logger import init_logger
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
encode_tokens)
from vllm.utils import flatten_2d_lists, full_groupby
from vllm.utils import (flatten_2d_lists, full_groupby,
get_allowed_kwarg_only_overrides)
from vllm.utils.jsontree import JSONTree, json_map_leaves
from .hasher import MultiModalHasher
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
@@ -34,6 +37,8 @@ if TYPE_CHECKING:
from transformers.feature_extraction_utils import BatchFeature
from transformers.processing_utils import ProcessorMixin
from vllm.config import ModelConfig
from .cache import BaseMultiModalProcessorCache
from .profiling import BaseDummyInputsBuilder
@@ -875,6 +880,222 @@ def find_mm_placeholders(
return dict(full_groupby_modality(it))
_T = TypeVar("_T")
_C = TypeVar("_C", bound="PretrainedConfig", default="PretrainedConfig")
_P = TypeVar("_P", bound="ProcessorMixin", default="ProcessorMixin")
@dataclass(frozen=True)
class InputProcessingContext:
"""
Contains information about the model which may be used to
modify the inputs.
"""
model_config: "ModelConfig"
"""The configuration of the model."""
tokenizer: AnyTokenizer
"""The tokenizer used to tokenize the inputs."""
@overload
def get_hf_config(self, /) -> "PretrainedConfig":
...
@overload
def get_hf_config(
self,
typ: Union[type[_C], tuple[type[_C], ...]],
/,
) -> _C:
...
def get_hf_config(
self,
typ: Optional[Union[type[Any], tuple[type[Any], ...]]] = None,
/,
) -> Any:
"""
Get the HuggingFace configuration
(`transformers.PretrainedConfig`) of the model,
additionally checking its type.
Raises:
TypeError: If the configuration is not of the specified type.
"""
if typ is None:
from transformers.configuration_utils import PretrainedConfig
typ = PretrainedConfig
hf_config = self.model_config.hf_config
if not isinstance(hf_config, typ):
raise TypeError("Invalid type of HuggingFace config. "
f"Expected type: {typ}, but "
f"found type: {type(hf_config)}")
return hf_config
def get_hf_image_processor_config(self) -> dict[str, Any]:
"""
Get the HuggingFace image processor configuration of the model.
"""
return self.model_config.hf_image_processor_config
def get_mm_config(self):
"""
Get the multimodal config of the model.
Raises:
RuntimeError: If the model is not a multimodal model.
"""
mm_config = self.model_config.multimodal_config
if mm_config is None:
raise RuntimeError("Not a multimodal model")
return mm_config
@overload
def get_hf_processor(self, /, **kwargs: object) -> "ProcessorMixin":
...
@overload
def get_hf_processor(
self,
typ: Union[type[_P], tuple[type[_P], ...]],
/,
**kwargs: object,
) -> _P:
...
def get_hf_processor(
self,
typ: Optional[Union[type[Any], tuple[type[Any], ...]]] = None,
/,
**kwargs: object,
) -> Any:
"""
Get the HuggingFace processor
(`transformers.ProcessorMixin`) of the model,
additionally checking its type.
Raises:
TypeError: If the processor is not of the specified type.
"""
if typ is None:
from transformers.processing_utils import ProcessorMixin
typ = ProcessorMixin
return cached_processor_from_config(
self.model_config,
processor_cls=typ,
tokenizer=self.tokenizer,
**kwargs,
)
def init_processor(
self,
typ: type[_T],
/,
**kwargs: object,
) -> _T:
"""
Initialize a HuggingFace-like processor class, merging the
keyword arguments with those in the model's configuration.
"""
mm_config = self.model_config.get_multimodal_config()
base_kwargs = mm_config.mm_processor_kwargs
if base_kwargs is None:
base_kwargs = {}
merged_kwargs = {**base_kwargs, **kwargs}
return typ(**merged_kwargs)
def _postprocess_output(
self,
output: JSONTree,
) -> JSONTree:
def _postprocess_one(x: object):
if isinstance(x, torch.Tensor): # noqa: SIM102
# This mimics the behavior of transformers.BatchFeature
if x.is_floating_point():
x = x.to(dtype=self.model_config.dtype)
return x
return json_map_leaves(_postprocess_one, output)
def call_hf_processor(
self,
hf_processor: "ProcessorMixin",
data: Mapping[str, object],
kwargs: Mapping[str, object] = {},
*,
num_tries: int = 1,
max_tries: int = 5,
) -> Union["BatchFeature", JSONTree]:
"""
Call `hf_processor` on the prompt `data`
(text, image, audio...) with configurable options `kwargs`.
"""
assert callable(hf_processor)
mm_config = self.model_config.get_multimodal_config()
merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs)
allowed_kwargs = get_allowed_kwarg_only_overrides(
hf_processor,
merged_kwargs,
requires_kw_only=False,
allow_var_kwargs=True,
)
try:
output = hf_processor(**data,
**allowed_kwargs,
return_tensors="pt")
except Exception as exc:
# See https://github.com/huggingface/tokenizers/issues/537
if (isinstance(exc, RuntimeError) and exc
and exc.args[0] == "Already borrowed"
and num_tries < max_tries):
logger.warning(
"Failed to acquire tokenizer in current thread. "
"Retrying (%d/%d)...", num_tries, max_tries)
time.sleep(0.5)
return self.call_hf_processor(
hf_processor,
data,
kwargs,
num_tries=num_tries + 1,
max_tries=max_tries,
)
msg = (f"Failed to apply {type(hf_processor).__name__} "
f"on data={data} with kwargs={allowed_kwargs}")
raise ValueError(msg) from exc
# this emulates output.to(dtype=self.model_config.dtype)
from transformers.feature_extraction_utils import BatchFeature
if isinstance(output, BatchFeature):
output_ = self._postprocess_output(output.data)
return BatchFeature(output_)
logger.warning_once(
"%s did not return `BatchFeature`. "
"Make sure to match the behaviour of `ProcessorMixin` when "
"implementing custom processors.",
type(hf_processor).__name__,
)
return self._postprocess_output(output)
class BaseProcessingInfo:
"""Base class to provide the information necessary for data processing."""