Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+61
-40
@@ -13,8 +13,9 @@ import typing
|
||||
import torch
|
||||
|
||||
|
||||
def compute_retained_tokens_count(video_size_thw: torch.LongTensor,
|
||||
spatial_merge_size: int, q: float) -> int:
|
||||
def compute_retained_tokens_count(
|
||||
video_size_thw: torch.LongTensor, spatial_merge_size: int, q: float
|
||||
) -> int:
|
||||
"""
|
||||
Compute the number of retained tokens for a given video.
|
||||
Method ensures that we retain all the tokens from the first frame
|
||||
@@ -66,23 +67,21 @@ def compute_retention_mask(
|
||||
)
|
||||
|
||||
# Core EVS
|
||||
similarity = torch.nn.functional.cosine_similarity(video_embeds[1:, ...],
|
||||
video_embeds[:-1, ...],
|
||||
dim=-1)
|
||||
similarity = torch.nn.functional.cosine_similarity(
|
||||
video_embeds[1:, ...], video_embeds[:-1, ...], dim=-1
|
||||
)
|
||||
dissimilarity = 1 - similarity
|
||||
|
||||
# Always ensure we include all tokens from the first frame
|
||||
dissimilarity = torch.cat(
|
||||
[255 * torch.ones_like(video_embeds[:1, :, :, 0]), dissimilarity],
|
||||
dim=0)
|
||||
[255 * torch.ones_like(video_embeds[:1, :, :, 0]), dissimilarity], dim=0
|
||||
)
|
||||
|
||||
dissimilarity_flat = dissimilarity.view(-1)
|
||||
order = torch.argsort(dissimilarity_flat,
|
||||
dim=-1,
|
||||
descending=True,
|
||||
stable=True)
|
||||
retain_num_tokens = compute_retained_tokens_count(video_size_thw,
|
||||
spatial_merge_size, q)
|
||||
order = torch.argsort(dissimilarity_flat, dim=-1, descending=True, stable=True)
|
||||
retain_num_tokens = compute_retained_tokens_count(
|
||||
video_size_thw, spatial_merge_size, q
|
||||
)
|
||||
topk_indices = order[:retain_num_tokens]
|
||||
|
||||
retention_mask = torch.zeros_like(dissimilarity_flat, dtype=torch.bool)
|
||||
@@ -119,18 +118,34 @@ def compute_mrope_for_media(
|
||||
llm_grid_h = video_size_thw[1] // spatial_merge_size
|
||||
llm_grid_w = video_size_thw[2] // spatial_merge_size
|
||||
|
||||
t_index = ((torch.arange(llm_grid_t).view(-1, 1).expand(
|
||||
-1, llm_grid_h * llm_grid_w).mul(
|
||||
tokens_per_second * video_second_per_grid)).long().flatten())
|
||||
h_index = (torch.arange(llm_grid_h).view(1, -1,
|
||||
1).expand(llm_grid_t, -1,
|
||||
llm_grid_w).flatten())
|
||||
w_index = (torch.arange(llm_grid_w).view(1, 1, -1).expand(
|
||||
llm_grid_t, llm_grid_h, -1).flatten())
|
||||
llm_grid_w = (torch.tensor([llm_grid_w
|
||||
]).view(1, 1,
|
||||
1).expand(llm_grid_t, llm_grid_h,
|
||||
llm_grid_w).flatten())
|
||||
t_index = (
|
||||
(
|
||||
torch.arange(llm_grid_t)
|
||||
.view(-1, 1)
|
||||
.expand(-1, llm_grid_h * llm_grid_w)
|
||||
.mul(tokens_per_second * video_second_per_grid)
|
||||
)
|
||||
.long()
|
||||
.flatten()
|
||||
)
|
||||
h_index = (
|
||||
torch.arange(llm_grid_h)
|
||||
.view(1, -1, 1)
|
||||
.expand(llm_grid_t, -1, llm_grid_w)
|
||||
.flatten()
|
||||
)
|
||||
w_index = (
|
||||
torch.arange(llm_grid_w)
|
||||
.view(1, 1, -1)
|
||||
.expand(llm_grid_t, llm_grid_h, -1)
|
||||
.flatten()
|
||||
)
|
||||
llm_grid_w = (
|
||||
torch.tensor([llm_grid_w])
|
||||
.view(1, 1, 1)
|
||||
.expand(llm_grid_t, llm_grid_h, llm_grid_w)
|
||||
.flatten()
|
||||
)
|
||||
|
||||
positions = torch.stack([t_index, h_index, w_index, llm_grid_w], dim=1)
|
||||
return positions
|
||||
@@ -183,7 +198,8 @@ def recompute_mrope_positions(
|
||||
|
||||
# Tensors
|
||||
positions: torch.LongTensor = typing.cast(
|
||||
torch.LongTensor, mrope_positions.clone()) # (3, N)
|
||||
torch.LongTensor, mrope_positions.clone()
|
||||
) # (3, N)
|
||||
N = input_ids.numel()
|
||||
|
||||
image_mask = input_ids.eq(image_token_id)
|
||||
@@ -193,8 +209,7 @@ def recompute_mrope_positions(
|
||||
|
||||
# Early exit: no media in this chunk
|
||||
if len(multimodal_positions) == 0:
|
||||
delta = (int((positions.max().item() + 1) -
|
||||
N) if positions.numel() else -N)
|
||||
delta = int((positions.max().item() + 1) - N) if positions.numel() else -N
|
||||
return positions, delta
|
||||
|
||||
total_mm_tokens = torch.count_nonzero(media_mask)
|
||||
@@ -203,12 +218,12 @@ def recompute_mrope_positions(
|
||||
# Early exit: we've updated positions for all media tokens
|
||||
# (and consequently - for all remaining text tokens)
|
||||
if seen_mm_tokens == total_mm_tokens:
|
||||
delta = (int((positions.max().item() + 1) -
|
||||
N) if positions.numel() else -N)
|
||||
delta = int((positions.max().item() + 1) - N) if positions.numel() else -N
|
||||
return positions, delta
|
||||
|
||||
vision_start_indices = (input_ids == vision_start_token_id).nonzero(
|
||||
as_tuple=True)[0]
|
||||
vision_start_indices = (input_ids == vision_start_token_id).nonzero(as_tuple=True)[
|
||||
0
|
||||
]
|
||||
|
||||
for mm_pos in multimodal_positions:
|
||||
# Each mm_pos can be a complete embedding for single media
|
||||
@@ -218,8 +233,9 @@ def recompute_mrope_positions(
|
||||
# - Current prefill chunk has no vision start indexes at all
|
||||
# - Vision start token appeared in previous prefill round
|
||||
# - Regular case
|
||||
seen_vision_start_indices = vision_start_indices[vision_start_indices <
|
||||
num_computed_tokens]
|
||||
seen_vision_start_indices = vision_start_indices[
|
||||
vision_start_indices < num_computed_tokens
|
||||
]
|
||||
|
||||
if len(seen_vision_start_indices):
|
||||
# If we have encountered some vision start indexes,
|
||||
@@ -228,19 +244,23 @@ def recompute_mrope_positions(
|
||||
# | TTTTTTTTTSVVVVVVVVVV|VVVVVVTTTTTTTTTTTTTTTT|
|
||||
last_vision_start_token = seen_vision_start_indices[-1]
|
||||
seem_mm_tokens_before_last_vision_start = torch.count_nonzero(
|
||||
media_mask[:last_vision_start_token])
|
||||
media_mask[:last_vision_start_token]
|
||||
)
|
||||
in_the_middle_of_media = (
|
||||
seen_mm_tokens > seem_mm_tokens_before_last_vision_start)
|
||||
seen_mm_tokens > seem_mm_tokens_before_last_vision_start
|
||||
)
|
||||
|
||||
if in_the_middle_of_media:
|
||||
mm_embeddings_seen = (seen_mm_tokens -
|
||||
seem_mm_tokens_before_last_vision_start)
|
||||
mm_embeddings_seen = (
|
||||
seen_mm_tokens - seem_mm_tokens_before_last_vision_start
|
||||
)
|
||||
global_mm_start = last_vision_start_token
|
||||
else:
|
||||
# We have completed previous mm_embedding part and
|
||||
# ready to start a new one
|
||||
next_vision_start_token = vision_start_indices[
|
||||
vision_start_indices >= num_computed_tokens][0]
|
||||
vision_start_indices >= num_computed_tokens
|
||||
][0]
|
||||
mm_embeddings_seen = 0
|
||||
global_mm_start = next_vision_start_token
|
||||
|
||||
@@ -248,7 +268,8 @@ def recompute_mrope_positions(
|
||||
# If there were no vision start indexes so far,
|
||||
# let's find first vision start index
|
||||
next_vision_start_token = vision_start_indices[
|
||||
vision_start_indices >= num_computed_tokens][0]
|
||||
vision_start_indices >= num_computed_tokens
|
||||
][0]
|
||||
|
||||
mm_embeddings_seen = 0
|
||||
global_mm_start = next_vision_start_token
|
||||
|
||||
Reference in New Issue
Block a user