[Mypy] Fix mypy typing for P models (#54169)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-09-04 20:47:11 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 5093e4844a
commit 784cac7c42
9 changed files with 126 additions and 45 deletions
+20 -14
View File
@@ -412,8 +412,8 @@ class SiglipVisionEmbeddings(nn.Module):
self.num_patches = (self.image_size // self.patch_size) ** 2
self.num_positions = self.num_patches
self.cache_position_embedding = dict()
self.cache_position_count = dict()
self.cache_position_embedding: dict[tuple[int, int], torch.Tensor] = {}
self.cache_position_count: dict[tuple[int, int], int] = {}
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
self.register_buffer(
@@ -469,7 +469,7 @@ class SiglipVisionEmbeddings(nn.Module):
if len(self.cache_position_embedding) >= max_cache:
min_hit_grid = min(
self.cache_position_count,
key=self.cache_position_count.get,
key=self.cache_position_count.__getitem__,
)
self.cache_position_count.pop(min_hit_grid)
self.cache_position_embedding.pop(min_hit_grid)
@@ -483,8 +483,7 @@ class SiglipVisionEmbeddings(nn.Module):
self,
pixel_values: torch.FloatTensor,
position_ids: torch.Tensor | None = None,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
image_grid_thw: Sequence[tuple[int, int, int]] | None = None,
interpolate_pos_encoding=False,
) -> torch.Tensor:
if pixel_values.dim() == 4:
@@ -508,6 +507,7 @@ class SiglipVisionEmbeddings(nn.Module):
start = 0
tmp_embeddings = list()
assert image_grid_thw is not None
for image_grid in image_grid_thw:
t, h, w = image_grid
end = start + t * h * w
@@ -864,7 +864,7 @@ class SiglipVisionTransformer(nn.Module):
height_position_ids: torch.Tensor | None = None,
width_position_ids: torch.Tensor | None = None,
cu_seqlens: torch.Tensor | None = None,
image_grid_thw: torch.Tensor | None = None,
image_grid_thw: Sequence[tuple[int, int, int]] | None = None,
) -> torch.Tensor:
hidden_states = self.embeddings(
pixel_values,
@@ -934,8 +934,7 @@ class SiglipVisionModel(nn.Module):
pixel_values,
interpolate_pos_encoding: bool = False,
position_ids: torch.Tensor | None = None,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
image_grid_thw: Sequence[tuple[int, int, int]] | None = None,
cu_seqlens: torch.Tensor | None = None,
) -> BaseModelOutputWithPooling:
return self.vision_model(
@@ -1029,17 +1028,24 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
offset = mm_feature.mm_position.offset
feature_data = mm_feature.data
assert feature_data is not None
if mm_feature.modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
grid_data = feature_data["image_grid_thw"].data
assert isinstance(grid_data, torch.Tensor)
t, h, w = grid_data.tolist()
assert t == 1, f"Image must have 1 frame, got {t}"
yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
elif mm_feature.modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
grid_data = feature_data["video_grid_thw"].data
assert isinstance(grid_data, torch.Tensor)
t, h, w = grid_data.tolist()
second_per_grid_ts = 1.0
if mm_feature.data.get("second_per_grid_ts", None):
second_per_grid_ts = mm_feature.data[
"second_per_grid_ts"
].data.item()
second_per_grid_item = feature_data.get("second_per_grid_ts")
if second_per_grid_item is not None:
second_per_grid_data = second_per_grid_item.data
assert isinstance(second_per_grid_data, torch.Tensor)
second_per_grid_ts = second_per_grid_data.item()
t_factor = second_per_grid_ts * tokens_per_second
yield (
offset,