[Model] Add support for Qwen2-VL video embeddings input & multiple image embeddings input with varied resolutions (#10221)
Signed-off-by: imkero <kerorek@outlook.com>
This commit is contained in:
@@ -79,7 +79,7 @@ logger = init_logger(__name__)
|
||||
|
||||
class Qwen2VLImagePixelInputs(TypedDict):
|
||||
type: Literal["pixel_values"]
|
||||
data: torch.Tensor
|
||||
pixel_values: torch.Tensor
|
||||
"""Shape:
|
||||
`(num_patches, num_channels * patch_size * patch_size)`
|
||||
"""
|
||||
@@ -92,9 +92,22 @@ class Qwen2VLImagePixelInputs(TypedDict):
|
||||
|
||||
class Qwen2VLImageEmbeddingInputs(TypedDict):
|
||||
type: Literal["image_embeds"]
|
||||
data: torch.Tensor
|
||||
"""Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
|
||||
`hidden_size` must match the hidden size of language model backbone.
|
||||
image_embeds: torch.Tensor
|
||||
"""Supported types:
|
||||
- List[`torch.Tensor`]: A list of tensors holding all images' features.
|
||||
Each tensor holds an image's features.
|
||||
- `torch.Tensor`: A tensor holding all images' features
|
||||
(concatenation of all images' feature tensors).
|
||||
|
||||
Tensor shape: `(num_image_features, hidden_size)`
|
||||
- `num_image_features` varies based on
|
||||
the number and resolution of the images.
|
||||
- `hidden_size` must match the hidden size of language model backbone.
|
||||
"""
|
||||
|
||||
image_grid_thw: torch.Tensor
|
||||
"""Shape: `(num_images, 3)`
|
||||
This should be in `(grid_t, grid_h, grid_w)` format.
|
||||
"""
|
||||
|
||||
|
||||
@@ -102,7 +115,8 @@ Qwen2VLImageInputs = Union[Qwen2VLImagePixelInputs,
|
||||
Qwen2VLImageEmbeddingInputs]
|
||||
|
||||
|
||||
class Qwen2VLVideoInputs(TypedDict):
|
||||
class Qwen2VLVideoPixelInputs(TypedDict):
|
||||
type: Literal["pixel_values_videos"]
|
||||
pixel_values_videos: torch.Tensor
|
||||
"""Shape:
|
||||
`(num_patches,
|
||||
@@ -116,6 +130,30 @@ class Qwen2VLVideoInputs(TypedDict):
|
||||
"""
|
||||
|
||||
|
||||
class Qwen2VLVideoEmbeddingInputs(TypedDict):
|
||||
type: Literal["video_embeds"]
|
||||
video_embeds: torch.Tensor
|
||||
"""Supported types:
|
||||
- List[`torch.Tensor`]: A list of tensors holding all videos' features.
|
||||
Each tensor holds an video's features.
|
||||
- `torch.Tensor`: A tensor holding all videos' features
|
||||
(concatenation of all videos' feature tensors).
|
||||
|
||||
Tensor shape: `(num_image_features, hidden_size)`
|
||||
- `num_image_features` varies based on
|
||||
the number and resolution of the videos.
|
||||
- `hidden_size` must match the hidden size of language model backbone.
|
||||
"""
|
||||
|
||||
video_grid_thw: torch.Tensor
|
||||
"""Shape: `(num_videos, 3)`
|
||||
This should be in `(grid_t, grid_h, grid_w)` format.
|
||||
"""
|
||||
|
||||
|
||||
Qwen2VLVideoInputs = Union[Qwen2VLVideoPixelInputs,
|
||||
Qwen2VLVideoEmbeddingInputs]
|
||||
|
||||
# === Vision Encoder === #
|
||||
|
||||
|
||||
@@ -585,6 +623,12 @@ def mm_input_mapper_for_qwen2_vl(
|
||||
"image_embeds": data.get("image_embeds"),
|
||||
"image_grid_thw": data.get("image_grid_thw"),
|
||||
})
|
||||
if data_type_key == "video" and isinstance(data, dict):
|
||||
return MultiModalKwargs({
|
||||
"video_embeds": data.get("video_embeds"),
|
||||
"video_grid_thw": data.get("video_grid_thw"),
|
||||
})
|
||||
|
||||
model_config = ctx.model_config
|
||||
# Handle mm processor kwargs; we pass these at creation time
|
||||
# because preprocess() in transformers doesn't expose them
|
||||
@@ -890,16 +934,33 @@ def input_processor_for_qwen2_vl(
|
||||
idx for idx, token in enumerate(prompt_token_ids)
|
||||
if token == hf_config.image_token_id
|
||||
]
|
||||
image_cnt = len(image_indices)
|
||||
embed_dim = image_inputs.get('image_embeds').size(0)
|
||||
assert embed_dim % image_cnt == 0
|
||||
num_pad_tokens = embed_dim // image_cnt
|
||||
|
||||
# ensure all image tokens have grid_thw
|
||||
assert \
|
||||
len(image_indices) == image_inputs["image_grid_thw"].size(0), \
|
||||
"image token num does not match image_grid_thw.shape"
|
||||
|
||||
image_counter = 0
|
||||
pad_token_counter = 0
|
||||
for idx, token in enumerate(prompt_token_ids):
|
||||
if idx in image_indices:
|
||||
grid_thw = image_inputs["image_grid_thw"][image_counter]
|
||||
grid_t, grid_h, grid_w = grid_thw
|
||||
num_pad_tokens = (grid_t * grid_h * grid_w //
|
||||
image_processor.merge_size //
|
||||
image_processor.merge_size)
|
||||
prompt_token_ids_with_image.extend([token] *
|
||||
num_pad_tokens)
|
||||
image_counter += 1
|
||||
pad_token_counter += num_pad_tokens
|
||||
else:
|
||||
prompt_token_ids_with_image.append(token)
|
||||
|
||||
# ensure all embeddings are used
|
||||
assert \
|
||||
pad_token_counter == image_inputs["image_embeds"].size(0), \
|
||||
"image_embeds.shape does not match image_grid_thw"
|
||||
|
||||
prompt_token_ids = prompt_token_ids_with_image
|
||||
else:
|
||||
prompt_token_ids = _expand_pad_tokens(image_inputs,
|
||||
@@ -912,14 +973,49 @@ def input_processor_for_qwen2_vl(
|
||||
max_pixels=max_pixels)
|
||||
|
||||
if video_inputs is not None:
|
||||
prompt_token_ids = _expand_pad_tokens(video_inputs,
|
||||
hf_config.video_token_id,
|
||||
make_batched_videos,
|
||||
"video",
|
||||
image_processor,
|
||||
prompt_token_ids,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels)
|
||||
if isinstance(video_inputs, dict):
|
||||
prompt_token_ids_with_video = []
|
||||
video_indices = [
|
||||
idx for idx, token in enumerate(prompt_token_ids)
|
||||
if token == hf_config.video_token_id
|
||||
]
|
||||
|
||||
# ensure all video tokens have grid_thw
|
||||
assert \
|
||||
len(video_indices) == video_inputs["video_grid_thw"].size(0), \
|
||||
"video token num does not match video_grid_thw.shape"
|
||||
|
||||
video_counter = 0
|
||||
pad_token_counter = 0
|
||||
for idx, token in enumerate(prompt_token_ids):
|
||||
if idx in video_indices:
|
||||
grid_thw = video_inputs["video_grid_thw"][video_counter]
|
||||
grid_t, grid_h, grid_w = grid_thw
|
||||
num_pad_tokens = (grid_t * grid_h * grid_w //
|
||||
image_processor.merge_size //
|
||||
image_processor.merge_size)
|
||||
prompt_token_ids_with_video.extend([token] *
|
||||
num_pad_tokens)
|
||||
video_counter += 1
|
||||
pad_token_counter += num_pad_tokens
|
||||
else:
|
||||
prompt_token_ids_with_video.append(token)
|
||||
|
||||
# ensure all embeddings are used
|
||||
assert \
|
||||
pad_token_counter == video_inputs["video_embeds"].size(0), \
|
||||
"video_embeds.shape does not match video_grid_thw"
|
||||
|
||||
prompt_token_ids = prompt_token_ids_with_video
|
||||
else:
|
||||
prompt_token_ids = _expand_pad_tokens(video_inputs,
|
||||
hf_config.video_token_id,
|
||||
make_batched_videos,
|
||||
"video",
|
||||
image_processor,
|
||||
prompt_token_ids,
|
||||
min_pixels=min_pixels,
|
||||
max_pixels=max_pixels)
|
||||
|
||||
prompt = inputs.get("prompt")
|
||||
if prompt is None:
|
||||
@@ -1051,49 +1147,71 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
|
||||
f"Got type: {type(pixel_values)}")
|
||||
|
||||
return Qwen2VLImagePixelInputs(type="pixel_values",
|
||||
data=pixel_values,
|
||||
pixel_values=pixel_values,
|
||||
image_grid_thw=image_grid_thw)
|
||||
|
||||
if image_embeds is not None:
|
||||
image_embeds = self._validate_and_reshape_mm_tensor(
|
||||
image_embeds, "image embeds")
|
||||
image_grid_thw = self._validate_and_reshape_mm_tensor(
|
||||
image_grid_thw, "image grid_thw")
|
||||
|
||||
if not isinstance(image_embeds, torch.Tensor):
|
||||
raise ValueError("Incorrect type of image embeddings. "
|
||||
f"Got type: {type(image_embeds)}")
|
||||
return Qwen2VLImageEmbeddingInputs(type="image_embeds",
|
||||
data=image_embeds)
|
||||
image_embeds=image_embeds,
|
||||
image_grid_thw=image_grid_thw)
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object) -> Optional[Qwen2VLVideoInputs]:
|
||||
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
|
||||
video_embeds = kwargs.pop("video_embeds", None)
|
||||
video_grid_thw = kwargs.pop("video_grid_thw", None)
|
||||
|
||||
if pixel_values_videos is None:
|
||||
if pixel_values_videos is None and video_embeds is None:
|
||||
return None
|
||||
|
||||
pixel_values_videos = self._validate_and_reshape_mm_tensor(
|
||||
pixel_values_videos, "video pixel values")
|
||||
video_grid_thw = self._validate_and_reshape_mm_tensor(
|
||||
video_grid_thw, "video grid_thw")
|
||||
if pixel_values_videos is not None:
|
||||
pixel_values_videos = self._validate_and_reshape_mm_tensor(
|
||||
pixel_values_videos, "video pixel values")
|
||||
video_grid_thw = self._validate_and_reshape_mm_tensor(
|
||||
video_grid_thw, "video grid_thw")
|
||||
|
||||
return Qwen2VLVideoInputs(
|
||||
pixel_values_videos=pixel_values_videos,
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
return Qwen2VLVideoPixelInputs(
|
||||
type="pixel_values_videos",
|
||||
pixel_values_videos=pixel_values_videos,
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
|
||||
if video_embeds is not None:
|
||||
video_embeds = self._validate_and_reshape_mm_tensor(
|
||||
video_embeds, "video embeds")
|
||||
video_grid_thw = self._validate_and_reshape_mm_tensor(
|
||||
video_grid_thw, "video grid_thw")
|
||||
|
||||
if not isinstance(video_embeds, torch.Tensor):
|
||||
raise ValueError("Incorrect type of video embeddings. "
|
||||
f"Got type: {type(video_embeds)}")
|
||||
return Qwen2VLVideoEmbeddingInputs(type="video_embeds",
|
||||
video_embeds=video_embeds,
|
||||
video_grid_thw=video_grid_thw)
|
||||
|
||||
def _process_image_input(self,
|
||||
image_input: Qwen2VLImageInputs) -> torch.Tensor:
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["data"].type(self.visual.dtype)
|
||||
return image_input["image_embeds"].type(self.visual.dtype)
|
||||
|
||||
pixel_values = image_input["data"].type(self.visual.dtype)
|
||||
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
|
||||
image_embeds = self.visual(pixel_values,
|
||||
grid_thw=image_input["image_grid_thw"])
|
||||
return image_embeds
|
||||
|
||||
def _process_video_input(self,
|
||||
video_input: Qwen2VLVideoInputs) -> torch.Tensor:
|
||||
if video_input["type"] == "video_embeds":
|
||||
return video_input["video_embeds"].type(self.visual.dtype)
|
||||
|
||||
pixel_values_videos = video_input["pixel_values_videos"].type(
|
||||
self.visual.dtype)
|
||||
video_embeds = self.visual(pixel_values_videos,
|
||||
|
||||
Reference in New Issue
Block a user