[Model] Support DP for ViT on Kimi-VL-A3B-Thinking-2506 (#23817)
Signed-off-by: Junhong <liujunhong11@huawei.com> Signed-off-by: LJH-LBJ <98734602+LJH-LBJ@users.noreply.github.com> Co-authored-by: Junhong <liujunhong11@huawei.com> Co-authored-by: LJH-LBJ <98734602+LJH-LBJ@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
co-authored by
Junhong
LJH-LBJ
Isotr0py
parent
cf91a89dd2
commit
a0e0efd6bd
+42
-15
@@ -9,7 +9,7 @@ from collections.abc import Iterable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from itertools import groupby
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
|
||||
from typing import TYPE_CHECKING, Any, Literal, Optional, TypeVar, Union
|
||||
from urllib.parse import ParseResult, urlparse
|
||||
from urllib.request import url2pathname
|
||||
|
||||
@@ -444,7 +444,6 @@ def run_dp_sharded_vision_model(image_input: torch.Tensor,
|
||||
Args:
|
||||
image_input (torch.Tensor): Image input tensor.
|
||||
vision_model (torch.nn.Module): Vision model.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: Output image embeddings
|
||||
"""
|
||||
@@ -542,6 +541,8 @@ def run_dp_sharded_mrope_vision_model(
|
||||
vision_model: torch.nn.Module,
|
||||
pixel_values: torch.Tensor,
|
||||
grid_thw_list: list[list[int]],
|
||||
*,
|
||||
rope_type: Literal["rope_3d", "rope_2d"],
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Run a vision model with data parallelism (DP) sharding.
|
||||
The function will shard the input image tensor on the
|
||||
@@ -552,6 +553,10 @@ def run_dp_sharded_mrope_vision_model(
|
||||
vision_model (torch.nn.Module): Vision model.
|
||||
pixel_values (torch.Tensor): Image/Video input tensor.
|
||||
grid_thw_list: List of grid dimensions for each image
|
||||
rope_type: Type of rope used in the vision model.
|
||||
Different rope types have different dimension to do ViT.
|
||||
"rope_3d" for 3D rope (e.g., Qwen2.5-VL)
|
||||
"rope_2d" for 2D rope (e.g., Kimi-VL)
|
||||
Returns:
|
||||
torch.Tensor: Output image embeddings
|
||||
|
||||
@@ -605,8 +610,12 @@ def run_dp_sharded_mrope_vision_model(
|
||||
device=pixel_values.device,
|
||||
dtype=pixel_values.dtype)
|
||||
# embed_dim_reduction_factor = 2 * 2
|
||||
embed_dim_reduction_factor = (vision_model.spatial_merge_size *
|
||||
vision_model.spatial_merge_size)
|
||||
if rope_type == "rope_2d":
|
||||
embed_dim_reduction_factor = (vision_model.merge_kernel_size[0] *
|
||||
vision_model.merge_kernel_size[1])
|
||||
else:
|
||||
embed_dim_reduction_factor = (vision_model.spatial_merge_size *
|
||||
vision_model.spatial_merge_size)
|
||||
|
||||
# Find the max length across all ranks
|
||||
# The output embedding of every DP rank has to be
|
||||
@@ -617,23 +626,42 @@ def run_dp_sharded_mrope_vision_model(
|
||||
local_grid_thw_list = [grid_thw_list[i] for i in image_idxs_local]
|
||||
|
||||
# Run the vision model on the local pixel_values_local
|
||||
if pixel_values_local.shape[0] > 0:
|
||||
image_embeds_local = vision_model(pixel_values_local,
|
||||
local_grid_thw_list)
|
||||
if rope_type == "rope_2d":
|
||||
if pixel_values_local.shape[0] > 0:
|
||||
image_embeds_local = vision_model(
|
||||
pixel_values_local, torch.tensor(local_grid_thw_list))
|
||||
if isinstance(image_embeds_local, list):
|
||||
image_embeds_local = torch.cat(image_embeds_local, dim=0)
|
||||
else:
|
||||
out_dim = getattr(vision_model.config, "hidden_size", None)
|
||||
image_embeds_local = torch.empty(
|
||||
(0, embed_dim_reduction_factor, out_dim),
|
||||
device=pixel_values.device,
|
||||
dtype=pixel_values.dtype)
|
||||
else:
|
||||
# Handle empty case
|
||||
image_embeds_local = torch.empty((0, vision_model.out_hidden_size),
|
||||
device=pixel_values.device,
|
||||
dtype=pixel_values.dtype)
|
||||
if pixel_values_local.shape[0] > 0:
|
||||
image_embeds_local = vision_model(pixel_values_local,
|
||||
local_grid_thw_list)
|
||||
else:
|
||||
# Handle empty case
|
||||
image_embeds_local = torch.empty((0, vision_model.out_hidden_size),
|
||||
device=pixel_values.device,
|
||||
dtype=pixel_values.dtype)
|
||||
|
||||
# Pad the output based on max_len_per_rank
|
||||
# for tensor_model_parallel_all_gather to work
|
||||
current_len = image_embeds_local.shape[0]
|
||||
if current_len < max_len_per_rank:
|
||||
padding_size = max_len_per_rank - current_len
|
||||
padding = torch.empty((padding_size, image_embeds_local.shape[1]),
|
||||
dtype=image_embeds_local.dtype,
|
||||
device=image_embeds_local.device)
|
||||
if rope_type == "rope_2d":
|
||||
padding = torch.empty((padding_size, image_embeds_local.shape[1],
|
||||
image_embeds_local.shape[2]),
|
||||
dtype=image_embeds_local.dtype,
|
||||
device=image_embeds_local.device)
|
||||
else:
|
||||
padding = torch.empty((padding_size, image_embeds_local.shape[1]),
|
||||
dtype=image_embeds_local.dtype,
|
||||
device=image_embeds_local.device)
|
||||
image_embeds_local_padded = torch.cat([image_embeds_local, padding],
|
||||
dim=0)
|
||||
else:
|
||||
@@ -674,7 +702,6 @@ def run_dp_sharded_mrope_vision_model(
|
||||
embed_start:embed_start + img_patches]
|
||||
embed_start += img_patches
|
||||
current_idx += count
|
||||
|
||||
out_embeddings = tuple(embed for embed in original_order_embeddings
|
||||
if embed is not None)
|
||||
assert len(out_embeddings) == len(
|
||||
|
||||
Reference in New Issue
Block a user