[Core][Frontend] Support Passing Multimodal Processor Kwargs (#8657)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
This commit is contained in:
+16
-3
@@ -14,7 +14,8 @@ from typing_extensions import TypeAlias
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.inputs import InputContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import JSONTree, is_list_of, json_map_leaves
|
||||
from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
|
||||
json_map_leaves)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -256,11 +257,20 @@ class MultiModalPlugin(ABC):
|
||||
model_cls, _ = get_model_architecture(model_config)
|
||||
|
||||
mapper = self._input_mappers.get(model_cls)
|
||||
# Only get processor kwargs at mapping time if we are not using the
|
||||
# input mapper; no overrides are used on the default here because they
|
||||
# should be passed to the huggingface resource at initialization time.
|
||||
if mapper is not None and mapper != self._default_input_mapper:
|
||||
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
|
||||
mapper, overrides=model_config.mm_processor_kwargs)
|
||||
else:
|
||||
mm_processor_kwargs = {}
|
||||
|
||||
if mapper is None:
|
||||
raise KeyError(f"No input mapper in {self} is registered for "
|
||||
f"model class {model_cls.__name__}.")
|
||||
|
||||
return mapper(InputContext(model_config), data)
|
||||
return mapper(InputContext(model_config), data, **mm_processor_kwargs)
|
||||
|
||||
@abstractmethod
|
||||
def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
|
||||
@@ -333,7 +343,10 @@ class MultiModalPlugin(ABC):
|
||||
f"for model class {model_cls.__name__} in {self}.")
|
||||
|
||||
if callable(max_mm_tokens):
|
||||
max_mm_tokens = max_mm_tokens(InputContext(model_config))
|
||||
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
|
||||
max_mm_tokens, overrides=model_config.mm_processor_kwargs)
|
||||
max_mm_tokens = max_mm_tokens(InputContext(model_config),
|
||||
**mm_processor_kwargs)
|
||||
|
||||
self._validate_max_multimodal_tokens(max_mm_tokens)
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ from PIL import Image
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.inputs.registry import InputContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.image_processor import get_image_processor
|
||||
from vllm.transformers_utils.processor import get_image_processor
|
||||
from vllm.utils import is_list_of
|
||||
|
||||
from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
|
||||
@@ -23,9 +23,14 @@ class ImagePlugin(MultiModalPlugin):
|
||||
return "image"
|
||||
|
||||
def _get_hf_image_processor(self, model_config: ModelConfig):
|
||||
mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
|
||||
else model_config.mm_processor_kwargs)
|
||||
# We don't explicitly check kwarg overrides to the HF class
|
||||
# since the automodel just takes kwargs, so we can't inspect it
|
||||
return cached_get_image_processor(
|
||||
model_config.model,
|
||||
trust_remote_code=model_config.trust_remote_code)
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
**mm_processor_kwargs)
|
||||
|
||||
def _default_input_mapper(
|
||||
self,
|
||||
@@ -37,6 +42,7 @@ class ImagePlugin(MultiModalPlugin):
|
||||
# PIL image
|
||||
if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
|
||||
image_processor = self._get_hf_image_processor(model_config)
|
||||
|
||||
if image_processor is None:
|
||||
raise RuntimeError("No HuggingFace processor is available "
|
||||
"to process the image object")
|
||||
|
||||
@@ -138,6 +138,15 @@ class MultiModalRegistry:
|
||||
"""
|
||||
Create an input mapper (see :meth:`map_input`) for a specific model.
|
||||
"""
|
||||
# NOTE - we currently make the assumption that if a model has multiple
|
||||
# supported modalities, they take the same kwargs. For the default,
|
||||
# this could be an issue in the future if it falls back to two HF
|
||||
# resources and we can't inspect the signature easily since it's
|
||||
# getting initialized through the autoclass.
|
||||
#
|
||||
# If this is a problem in the future, we should revisit it, but since
|
||||
# it potentially introduces a lot of complexity for a currently
|
||||
# uncommon case, we do not for simplicity of both use & implementation
|
||||
return functools.partial(self.map_input, model_config)
|
||||
|
||||
def register_max_multimodal_tokens(
|
||||
|
||||
@@ -6,7 +6,7 @@ import numpy as np
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.inputs.registry import InputContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.image_processor import get_video_processor
|
||||
from vllm.transformers_utils.processor import get_video_processor
|
||||
from vllm.transformers_utils.tokenizer import get_tokenizer
|
||||
from vllm.utils import is_list_of
|
||||
|
||||
@@ -37,9 +37,14 @@ class VideoPlugin(ImagePlugin):
|
||||
return "video"
|
||||
|
||||
def _get_hf_video_processor(self, model_config: ModelConfig):
|
||||
mm_processor_kwargs = ({} if model_config.mm_processor_kwargs is None
|
||||
else model_config.mm_processor_kwargs)
|
||||
# We don't explicitly check kwarg overrides to the HF class
|
||||
# since the automodel just takes kwargs, so we can't inspect it
|
||||
return cached_get_video_processor(
|
||||
model_config.model,
|
||||
trust_remote_code=model_config.trust_remote_code)
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
**mm_processor_kwargs)
|
||||
|
||||
def _default_input_mapper(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user