[Mypy] Fix mypy typing for L models (#54177)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar
|
||||
from typing import Annotated, Literal, Protocol, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -159,14 +159,14 @@ class LlavaMultiModalProjector(nn.Module):
|
||||
|
||||
|
||||
class LlavaLikeConfig(Protocol):
|
||||
vision_config: Final[PretrainedConfig]
|
||||
image_token_index: Final[int]
|
||||
vision_feature_select_strategy: Final[str]
|
||||
vision_feature_layer: Final[int | list[int]]
|
||||
vision_config: PretrainedConfig
|
||||
image_token_index: int
|
||||
vision_feature_select_strategy: str
|
||||
vision_feature_layer: int | list[int]
|
||||
|
||||
|
||||
class LlavaLikeProcessor(Protocol):
|
||||
image_token: Final[str]
|
||||
image_token: str
|
||||
|
||||
|
||||
class BaseLlavaProcessingInfo(BaseProcessingInfo):
|
||||
@@ -286,6 +286,7 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
num_image_tokens = images.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(images, ImageProcessorItems)
|
||||
image_size = images.get_image_size(item_idx)
|
||||
num_image_tokens = self.info.get_num_image_tokens(
|
||||
image_width=image_size.width,
|
||||
@@ -372,6 +373,7 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn
|
||||
image_end_id = vocab[processor.image_end_token]
|
||||
|
||||
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
|
||||
assert isinstance(hf_config, LlavaConfig)
|
||||
encoder_info = PixtralHFEncoderInfo(hf_config)
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
@@ -642,7 +644,7 @@ class LlavaForConditionalGeneration(
|
||||
self,
|
||||
image_input: LlavaImageInputs,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
if image_input["type"] == "image_embeds":
|
||||
if isinstance(image_input, LlavaImageEmbeddingInputs):
|
||||
return image_input["data"]
|
||||
|
||||
image_features = self._process_image_pixels(image_input)
|
||||
|
||||
Reference in New Issue
Block a user