[Misc] Clean up type annotation for SupportsMultiModal (#14794)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -5,7 +5,7 @@ from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,
|
||||
|
||||
import torch
|
||||
from torch import Tensor
|
||||
from typing_extensions import TypeIs, TypeVar
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
@@ -20,7 +20,14 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
T = TypeVar("T", default=Union[list[Tensor], Tensor, tuple[Tensor, ...]])
|
||||
MultiModalEmbeddings = Union[list[Tensor], Tensor, tuple[Tensor, ...]]
|
||||
"""
|
||||
The output embeddings must be one of the following formats:
|
||||
|
||||
- A list or tuple of 2D tensors, where each tensor corresponds to
|
||||
each input multimodal data item (e.g, image).
|
||||
- A single 3D tensor, with the batch dimension grouping the 2D tensors.
|
||||
"""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
@@ -36,17 +43,12 @@ class SupportsMultiModal(Protocol):
|
||||
MRO of your model class.
|
||||
"""
|
||||
|
||||
def get_multimodal_embeddings(self, **kwargs) -> T:
|
||||
def get_multimodal_embeddings(
|
||||
self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
|
||||
"""
|
||||
Returns multimodal embeddings generated from multimodal kwargs
|
||||
to be merged with text embeddings.
|
||||
|
||||
The output embeddings must be one of the following formats:
|
||||
|
||||
- A list or tuple of 2D tensors, where each tensor corresponds to
|
||||
each input multimodal data item (e.g, image).
|
||||
- A single 3D tensor, with the batch dimension grouping the 2D tensors.
|
||||
|
||||
Note:
|
||||
The returned multimodal embeddings must be in the same order as
|
||||
the appearances of their corresponding multimodal data item in the
|
||||
@@ -60,7 +62,7 @@ class SupportsMultiModal(Protocol):
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: Tensor,
|
||||
multimodal_embeddings: Optional[T] = None,
|
||||
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
|
||||
attn_metadata: Optional["AttentionMetadata"] = None,
|
||||
) -> Tensor:
|
||||
...
|
||||
@@ -69,7 +71,7 @@ class SupportsMultiModal(Protocol):
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: Tensor,
|
||||
multimodal_embeddings: Optional[T] = None,
|
||||
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
|
||||
) -> Tensor:
|
||||
"""
|
||||
Returns the input embeddings merged from the text embeddings from
|
||||
|
||||
Reference in New Issue
Block a user