[Misc] Clean up type annotation for SupportsMultiModal (#14794)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-03-14 00:59:56 -07:00
committed by GitHub
parent 09269b3127
commit 601bd3268e
27 changed files with 121 additions and 141 deletions
+13 -11
View File
@@ -5,7 +5,7 @@ from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,
import torch
from torch import Tensor
from typing_extensions import TypeIs, TypeVar
from typing_extensions import TypeIs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.base_config import (
@@ -20,7 +20,14 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
T = TypeVar("T", default=Union[list[Tensor], Tensor, tuple[Tensor, ...]])
MultiModalEmbeddings = Union[list[Tensor], Tensor, tuple[Tensor, ...]]
"""
The output embeddings must be one of the following formats:
- A list or tuple of 2D tensors, where each tensor corresponds to
each input multimodal data item (e.g, image).
- A single 3D tensor, with the batch dimension grouping the 2D tensors.
"""
@runtime_checkable
@@ -36,17 +43,12 @@ class SupportsMultiModal(Protocol):
MRO of your model class.
"""
def get_multimodal_embeddings(self, **kwargs) -> T:
def get_multimodal_embeddings(
self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
"""
Returns multimodal embeddings generated from multimodal kwargs
to be merged with text embeddings.
The output embeddings must be one of the following formats:
- A list or tuple of 2D tensors, where each tensor corresponds to
each input multimodal data item (e.g, image).
- A single 3D tensor, with the batch dimension grouping the 2D tensors.
Note:
The returned multimodal embeddings must be in the same order as
the appearances of their corresponding multimodal data item in the
@@ -60,7 +62,7 @@ class SupportsMultiModal(Protocol):
def get_input_embeddings(
self,
input_ids: Tensor,
multimodal_embeddings: Optional[T] = None,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
attn_metadata: Optional["AttentionMetadata"] = None,
) -> Tensor:
...
@@ -69,7 +71,7 @@ class SupportsMultiModal(Protocol):
def get_input_embeddings(
self,
input_ids: Tensor,
multimodal_embeddings: Optional[T] = None,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
) -> Tensor:
"""
Returns the input embeddings merged from the text embeddings from