[Frontend] decrease import time of vllm.multimodal (#18031)

Co-authored-by: Aaron Pham <Aaronpham0103@gmail.com>
This commit is contained in:
David Xia
2025-05-14 15:43:32 -07:00
committed by GitHub
co-authored by Aaron Pham
parent 856865008e
commit 749f792553
3 changed files with 45 additions and 34 deletions
+8 -5
View File
@@ -13,7 +13,6 @@ from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
TypeVar, Union, cast)
import torch
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
from typing_extensions import assert_never
from vllm.inputs import InputProcessingContext
@@ -31,6 +30,10 @@ from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,
MultiModalDataParser)
if TYPE_CHECKING:
from transformers.configuration_utils import PretrainedConfig
from transformers.feature_extraction_utils import BatchFeature
from transformers.processing_utils import ProcessorMixin
from .profiling import BaseDummyInputsBuilder
logger = init_logger(__name__)
@@ -1047,10 +1050,10 @@ class BaseProcessingInfo:
def get_tokenizer(self) -> AnyTokenizer:
return self.ctx.tokenizer
def get_hf_config(self) -> PretrainedConfig:
def get_hf_config(self) -> "PretrainedConfig":
return self.ctx.get_hf_config()
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
def get_hf_processor(self, **kwargs: object) -> "ProcessorMixin":
"""
Subclasses can override this method to handle
specific kwargs from model config or user inputs.
@@ -1165,7 +1168,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
@abstractmethod
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_inputs: "BatchFeature",
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
"""Given the HF-processed data, output the metadata of each field."""
@@ -1222,7 +1225,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# This refers to the data to be passed to HF processor.
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
) -> BatchFeature:
) -> "BatchFeature":
"""
Call the HF processor on the prompt text and
associated multi-modal data.