[Model] Merge SupportsMultiModalWithRawInput with SupportsMultiModal (#23749)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -52,6 +52,12 @@ class SupportsMultiModal(Protocol):
|
||||
MRO of your model class.
|
||||
"""
|
||||
|
||||
supports_multimodal_raw_input_only: ClassVar[bool] = False
|
||||
"""
|
||||
A flag that indicates this model supports multi-modal inputs and processes
|
||||
them in their raw form and not embeddings.
|
||||
"""
|
||||
|
||||
supports_encoder_tp_data: ClassVar[bool] = False
|
||||
"""
|
||||
A flag that indicates whether this model supports
|
||||
@@ -143,45 +149,16 @@ def supports_multimodal(
|
||||
return getattr(model, "supports_multimodal", False)
|
||||
|
||||
|
||||
def supports_multimodal_raw_input_only(
|
||||
model: Union[type[object], object]) -> bool:
|
||||
return getattr(model, "supports_multimodal_raw_input_only", False)
|
||||
|
||||
|
||||
def supports_multimodal_encoder_tp_data(
|
||||
model: Union[type[object], object]) -> bool:
|
||||
return getattr(model, "supports_encoder_tp_data", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsMultiModalWithRawInput(SupportsMultiModal, Protocol):
|
||||
"""The interface required for all multi-modal models."""
|
||||
|
||||
supports_multimodal_raw_input: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
A flag that indicates this model supports multi-modal inputs and processes
|
||||
them in their raw form and not embeddings.
|
||||
|
||||
Note:
|
||||
There is no need to redefine this flag if this class is in the
|
||||
MRO of your model class.
|
||||
"""
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal_raw_input(
|
||||
model: object) -> TypeIs[SupportsMultiModalWithRawInput]:
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_multimodal_raw_input(
|
||||
model: type[object]) -> TypeIs[type[SupportsMultiModalWithRawInput]]:
|
||||
...
|
||||
|
||||
|
||||
def supports_multimodal_raw_input(
|
||||
model: Union[type[object], object]
|
||||
) -> Union[TypeIs[type[SupportsMultiModalWithRawInput]],
|
||||
TypeIs[SupportsMultiModalWithRawInput]]:
|
||||
return getattr(model, "supports_multimodal_raw_input", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsScoreTemplate(Protocol):
|
||||
"""The interface required for all models that support score template."""
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import (IsAttentionFree, MultiModalEmbeddings,
|
||||
SupportsMultiModalWithRawInput)
|
||||
SupportsMultiModal)
|
||||
from .interfaces_base import default_pooling_type
|
||||
|
||||
|
||||
@@ -174,10 +174,10 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
|
||||
info=PrithviGeoSpatialMAEProcessingInfo,
|
||||
dummy_inputs=PrithviGeoSpatialMAEInputBuilder,
|
||||
)
|
||||
class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree,
|
||||
SupportsMultiModalWithRawInput):
|
||||
class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal):
|
||||
"""Prithvi Masked Autoencoder"""
|
||||
|
||||
supports_multimodal_raw_input_only = True
|
||||
is_pooling_model = True
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -29,7 +29,7 @@ from .interfaces import (has_inner_state, has_noops, is_attention_free,
|
||||
is_hybrid, supports_cross_encoding,
|
||||
supports_multimodal,
|
||||
supports_multimodal_encoder_tp_data,
|
||||
supports_multimodal_raw_input, supports_pp,
|
||||
supports_multimodal_raw_input_only, supports_pp,
|
||||
supports_transcription, supports_v0_only)
|
||||
from .interfaces_base import (get_default_pooling_type, is_pooling_model,
|
||||
is_text_generation_model)
|
||||
@@ -326,7 +326,7 @@ class _ModelInfo:
|
||||
default_pooling_type: str
|
||||
supports_cross_encoding: bool
|
||||
supports_multimodal: bool
|
||||
supports_multimodal_raw_input: bool
|
||||
supports_multimodal_raw_input_only: bool
|
||||
supports_multimodal_encoder_tp_data: bool
|
||||
supports_pp: bool
|
||||
has_inner_state: bool
|
||||
@@ -346,7 +346,8 @@ class _ModelInfo:
|
||||
default_pooling_type=get_default_pooling_type(model),
|
||||
supports_cross_encoding=supports_cross_encoding(model),
|
||||
supports_multimodal=supports_multimodal(model),
|
||||
supports_multimodal_raw_input=supports_multimodal_raw_input(model),
|
||||
supports_multimodal_raw_input_only=
|
||||
supports_multimodal_raw_input_only(model),
|
||||
supports_multimodal_encoder_tp_data=
|
||||
supports_multimodal_encoder_tp_data(model),
|
||||
supports_pp=supports_pp(model),
|
||||
@@ -743,13 +744,13 @@ class _ModelRegistry:
|
||||
model_cls, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_cls.supports_multimodal
|
||||
|
||||
def supports_multimodal_raw_input(
|
||||
def is_multimodal_raw_input_only_model(
|
||||
self,
|
||||
architectures: Union[str, list[str]],
|
||||
model_config: ModelConfig,
|
||||
) -> bool:
|
||||
model_cls, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_cls.supports_multimodal_raw_input
|
||||
return model_cls.supports_multimodal_raw_input_only
|
||||
|
||||
def is_pp_supported_model(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user