[Mypy] Fix mypy typing for P models (#54169)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
5093e4844a
commit
784cac7c42
@@ -412,8 +412,8 @@ class SiglipVisionEmbeddings(nn.Module):
|
||||
|
||||
self.num_patches = (self.image_size // self.patch_size) ** 2
|
||||
self.num_positions = self.num_patches
|
||||
self.cache_position_embedding = dict()
|
||||
self.cache_position_count = dict()
|
||||
self.cache_position_embedding: dict[tuple[int, int], torch.Tensor] = {}
|
||||
self.cache_position_count: dict[tuple[int, int], int] = {}
|
||||
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
|
||||
|
||||
self.register_buffer(
|
||||
@@ -469,7 +469,7 @@ class SiglipVisionEmbeddings(nn.Module):
|
||||
if len(self.cache_position_embedding) >= max_cache:
|
||||
min_hit_grid = min(
|
||||
self.cache_position_count,
|
||||
key=self.cache_position_count.get,
|
||||
key=self.cache_position_count.__getitem__,
|
||||
)
|
||||
self.cache_position_count.pop(min_hit_grid)
|
||||
self.cache_position_embedding.pop(min_hit_grid)
|
||||
@@ -483,8 +483,7 @@ class SiglipVisionEmbeddings(nn.Module):
|
||||
self,
|
||||
pixel_values: torch.FloatTensor,
|
||||
position_ids: torch.Tensor | None = None,
|
||||
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
|
||||
| None = None,
|
||||
image_grid_thw: Sequence[tuple[int, int, int]] | None = None,
|
||||
interpolate_pos_encoding=False,
|
||||
) -> torch.Tensor:
|
||||
if pixel_values.dim() == 4:
|
||||
@@ -508,6 +507,7 @@ class SiglipVisionEmbeddings(nn.Module):
|
||||
|
||||
start = 0
|
||||
tmp_embeddings = list()
|
||||
assert image_grid_thw is not None
|
||||
for image_grid in image_grid_thw:
|
||||
t, h, w = image_grid
|
||||
end = start + t * h * w
|
||||
@@ -864,7 +864,7 @@ class SiglipVisionTransformer(nn.Module):
|
||||
height_position_ids: torch.Tensor | None = None,
|
||||
width_position_ids: torch.Tensor | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
image_grid_thw: torch.Tensor | None = None,
|
||||
image_grid_thw: Sequence[tuple[int, int, int]] | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.embeddings(
|
||||
pixel_values,
|
||||
@@ -934,8 +934,7 @@ class SiglipVisionModel(nn.Module):
|
||||
pixel_values,
|
||||
interpolate_pos_encoding: bool = False,
|
||||
position_ids: torch.Tensor | None = None,
|
||||
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
|
||||
| None = None,
|
||||
image_grid_thw: Sequence[tuple[int, int, int]] | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
) -> BaseModelOutputWithPooling:
|
||||
return self.vision_model(
|
||||
@@ -1029,17 +1028,24 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
|
||||
tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
offset = mm_feature.mm_position.offset
|
||||
feature_data = mm_feature.data
|
||||
assert feature_data is not None
|
||||
if mm_feature.modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
grid_data = feature_data["image_grid_thw"].data
|
||||
assert isinstance(grid_data, torch.Tensor)
|
||||
t, h, w = grid_data.tolist()
|
||||
assert t == 1, f"Image must have 1 frame, got {t}"
|
||||
yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
|
||||
elif mm_feature.modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
grid_data = feature_data["video_grid_thw"].data
|
||||
assert isinstance(grid_data, torch.Tensor)
|
||||
t, h, w = grid_data.tolist()
|
||||
second_per_grid_ts = 1.0
|
||||
if mm_feature.data.get("second_per_grid_ts", None):
|
||||
second_per_grid_ts = mm_feature.data[
|
||||
"second_per_grid_ts"
|
||||
].data.item()
|
||||
second_per_grid_item = feature_data.get("second_per_grid_ts")
|
||||
if second_per_grid_item is not None:
|
||||
second_per_grid_data = second_per_grid_item.data
|
||||
assert isinstance(second_per_grid_data, torch.Tensor)
|
||||
second_per_grid_ts = second_per_grid_data.item()
|
||||
t_factor = second_per_grid_ts * tokens_per_second
|
||||
yield (
|
||||
offset,
|
||||
|
||||
Reference in New Issue
Block a user