[Mypy] Fix mypy typing for L models (#54177)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-04 07:50:43 -04:00
committed by GitHub
parent 3ff4f02dfe
commit c615b1fd67
19 changed files with 255 additions and 89 deletions
+9 -7
View File
@@ -3,7 +3,7 @@
from abc import abstractmethod
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar
from typing import Annotated, Literal, Protocol, TypeAlias, TypeVar
import torch
import torch.nn as nn
@@ -159,14 +159,14 @@ class LlavaMultiModalProjector(nn.Module):
class LlavaLikeConfig(Protocol):
vision_config: Final[PretrainedConfig]
image_token_index: Final[int]
vision_feature_select_strategy: Final[str]
vision_feature_layer: Final[int | list[int]]
vision_config: PretrainedConfig
image_token_index: int
vision_feature_select_strategy: str
vision_feature_layer: int | list[int]
class LlavaLikeProcessor(Protocol):
image_token: Final[str]
image_token: str
class BaseLlavaProcessingInfo(BaseProcessingInfo):
@@ -286,6 +286,7 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
assert isinstance(images, ImageProcessorItems)
image_size = images.get_image_size(item_idx)
num_image_tokens = self.info.get_num_image_tokens(
image_width=image_size.width,
@@ -372,6 +373,7 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn
image_end_id = vocab[processor.image_end_token]
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
assert isinstance(hf_config, LlavaConfig)
encoder_info = PixtralHFEncoderInfo(hf_config)
def get_replacement(item_idx: int):
@@ -642,7 +644,7 @@ class LlavaForConditionalGeneration(
self,
image_input: LlavaImageInputs,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
if image_input["type"] == "image_embeds":
if isinstance(image_input, LlavaImageEmbeddingInputs):
return image_input["data"]
image_features = self._process_image_pixels(image_input)