[Model] merged input processor for Phi-3-Vision models (#10977)

Signed-off-by: Isotr0py <2037008807@qq.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Isotr0py
2024-12-09 12:55:10 -08:00
committed by GitHub
co-authored by Cyrus Leung
parent ca871491ed
commit a811dd6608
7 changed files with 234 additions and 408 deletions
+25 -4
View File
@@ -3,7 +3,8 @@ from abc import ABC, abstractmethod
from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
from dataclasses import dataclass
from functools import lru_cache
from typing import Any, Generic, NamedTuple, Optional, Protocol, TypeVar, Union
from typing import (Any, Dict, Generic, NamedTuple, Optional, Protocol,
TypeVar, Union, cast)
import torch
from transformers import BatchFeature, ProcessorMixin
@@ -11,7 +12,8 @@ from typing_extensions import TypeAlias, TypedDict
from vllm.inputs import DummyData, InputProcessingContext
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
from vllm.utils import (flatten_2d_lists, full_groupby, is_list_of,
resolve_mm_processor_kwargs)
from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
@@ -543,8 +545,14 @@ class BaseMultiModalProcessor(ABC):
self.ctx = ctx
self.metadata = metadata
self.init_mm_processor_kwargs = (ctx.model_config.mm_processor_kwargs
or {})
def _get_hf_processor(self) -> ProcessorMixin:
def _get_hf_processor(
self,
**mm_processor_kwargs: Mapping[str, object],
) -> ProcessorMixin:
# by default, we won't pass any kwargs to the processor initialization
return self.ctx.get_hf_processor()
def _get_tokenizer(self) -> AnyTokenizer:
@@ -581,7 +589,13 @@ class BaseMultiModalProcessor(ABC):
mm_data: MultiModalDataDict,
mm_processor_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_processor = self._get_hf_processor()
# some mm_processor_kwargs may be used in processor initialization
# instead of processor call
processor_init_kwargs = {
**self.init_mm_processor_kwargs,
**mm_processor_kwargs,
}
hf_processor = self._get_hf_processor(**processor_init_kwargs)
processor_data = dict[str, Any]()
passthrough_data = dict[str, Any]()
@@ -601,6 +615,13 @@ class BaseMultiModalProcessor(ABC):
else:
processor_data[k] = v
# filter mm_processor_kwargs used in processor call
mm_processor_kwargs = resolve_mm_processor_kwargs(
self.init_mm_processor_kwargs,
cast(Dict[str, Any], mm_processor_kwargs),
hf_processor,
)
try:
hf_inputs = hf_processor(
text=prompt, # type: ignore