Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
""" PyTorch Ovis model."""
|
||||
"""PyTorch Ovis model."""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from functools import partial
|
||||
from typing import Literal, Optional, TypedDict, Union
|
||||
@@ -13,18 +14,26 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.models.ovis import (OvisImagePatchInputs,
|
||||
VisualEmbedding)
|
||||
from vllm.model_executor.models.ovis import OvisImagePatchInputs, VisualEmbedding
|
||||
from vllm.model_executor.models.siglip2navit import Siglip2NavitModel
|
||||
from vllm.model_executor.models.utils import (AutoWeightsLoader, flatten_bn,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
flatten_bn,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
|
||||
MultiModalKwargsItems)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalDataDict,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, PromptReplacement)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor
|
||||
@@ -71,12 +80,14 @@ class OvisVideoPatchInputs(TypedDict):
|
||||
|
||||
|
||||
def _ovis2_5_field_config():
|
||||
return dict(pixel_values=MultiModalFieldConfig.batched("image"),
|
||||
grids=MultiModalFieldConfig.batched("image"),
|
||||
indicator_tokens=MultiModalFieldConfig.batched("image"),
|
||||
video_pixel_values=MultiModalFieldConfig.batched("video"),
|
||||
video_indicator_tokens=MultiModalFieldConfig.batched("video"),
|
||||
video_grids=MultiModalFieldConfig.batched("video"))
|
||||
return dict(
|
||||
pixel_values=MultiModalFieldConfig.batched("image"),
|
||||
grids=MultiModalFieldConfig.batched("image"),
|
||||
indicator_tokens=MultiModalFieldConfig.batched("image"),
|
||||
video_pixel_values=MultiModalFieldConfig.batched("video"),
|
||||
video_indicator_tokens=MultiModalFieldConfig.batched("video"),
|
||||
video_grids=MultiModalFieldConfig.batched("video"),
|
||||
)
|
||||
|
||||
|
||||
class VisualTokenizer(torch.nn.Module):
|
||||
@@ -108,7 +119,9 @@ class VisualTokenizer(torch.nn.Module):
|
||||
head_dim,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
), torch.nn.LayerNorm(head_dim))
|
||||
),
|
||||
torch.nn.LayerNorm(head_dim),
|
||||
)
|
||||
|
||||
def _init_backbone(
|
||||
self,
|
||||
@@ -119,12 +132,13 @@ class VisualTokenizer(torch.nn.Module):
|
||||
):
|
||||
model_type = config.model_type
|
||||
if model_type == "siglip2_navit":
|
||||
return Siglip2NavitModel(config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
use_data_parallel=use_data_parallel)
|
||||
raise ValueError(
|
||||
f"Unsupported visual tokenizer model_type: {model_type}")
|
||||
return Siglip2NavitModel(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
use_data_parallel=use_data_parallel,
|
||||
)
|
||||
raise ValueError(f"Unsupported visual tokenizer model_type: {model_type}")
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
@@ -135,22 +149,22 @@ class VisualTokenizer(torch.nn.Module):
|
||||
return next(self.head.parameters()).device
|
||||
|
||||
def tokenize(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
tokens = torch.softmax(logits, dim=-1,
|
||||
dtype=torch.float32).to(logits.dtype)
|
||||
tokens = torch.softmax(logits, dim=-1, dtype=torch.float32).to(logits.dtype)
|
||||
return tokens
|
||||
|
||||
def encode(self, pixel_values: torch.Tensor,
|
||||
grid_thws: torch.Tensor) -> torch.Tensor:
|
||||
def encode(
|
||||
self, pixel_values: torch.Tensor, grid_thws: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
features = self.vit(pixel_values, grid_thws)
|
||||
# refer to qwen2.5-vl patchmerger
|
||||
seq_len, _ = features.shape
|
||||
features = features.reshape(seq_len // (self.config.hidden_stride**2),
|
||||
-1)
|
||||
features = features.reshape(seq_len // (self.config.hidden_stride**2), -1)
|
||||
|
||||
return features
|
||||
|
||||
def forward(self, pixel_values: torch.Tensor,
|
||||
grid_thws: torch.Tensor) -> torch.Tensor:
|
||||
def forward(
|
||||
self, pixel_values: torch.Tensor, grid_thws: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
features = self.encode(pixel_values, grid_thws)
|
||||
logits = self.head(features)
|
||||
tokens = self.tokenize(logits)
|
||||
@@ -167,7 +181,6 @@ class VisualTokenizer(torch.nn.Module):
|
||||
|
||||
|
||||
class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config()
|
||||
|
||||
@@ -220,8 +233,9 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_max_image_tokens(self) -> int:
|
||||
target_width, target_height = self.get_image_size_with_most_features()
|
||||
return self.get_num_image_tokens(image_width=target_width,
|
||||
image_height=target_height)
|
||||
return self.get_num_image_tokens(
|
||||
image_width=target_width, image_height=target_height
|
||||
)
|
||||
|
||||
def _get_max_video_frames(self, max_tokens: int) -> int:
|
||||
target_width, target_height = self.get_image_size_with_most_features()
|
||||
@@ -247,8 +261,7 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
||||
max_images = mm_counts.get("image", 0)
|
||||
max_videos = mm_counts.get("video", 0)
|
||||
max_image_tokens = self.get_max_image_tokens() * max_images
|
||||
max_total_frames = self._get_max_video_frames(seq_len -
|
||||
max_image_tokens)
|
||||
max_total_frames = self._get_max_video_frames(seq_len - max_image_tokens)
|
||||
max_frames_per_video = max_total_frames // max(max_videos, 1)
|
||||
return max(max_frames_per_video, 1)
|
||||
|
||||
@@ -260,9 +273,9 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
||||
num_frames: int,
|
||||
image_processor: Optional[BaseImageProcessor],
|
||||
) -> int:
|
||||
num_video_tokens = self.get_num_image_tokens(image_width=image_width,
|
||||
image_height=image_height,
|
||||
num_frames=num_frames)
|
||||
num_video_tokens = self.get_num_image_tokens(
|
||||
image_width=image_width, image_height=image_height, num_frames=num_frames
|
||||
)
|
||||
return num_video_tokens
|
||||
|
||||
def get_max_video_tokens(
|
||||
@@ -274,14 +287,12 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
|
||||
return self.get_num_video_tokens(
|
||||
image_width=target_width,
|
||||
image_height=target_height,
|
||||
num_frames=self.get_num_frames_with_most_features(
|
||||
seq_len, mm_counts),
|
||||
num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
|
||||
image_processor=None,
|
||||
)
|
||||
|
||||
|
||||
class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
|
||||
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
@@ -296,48 +307,47 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
|
||||
num_images = mm_counts.get("image", 0)
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
|
||||
target_width, target_height = \
|
||||
self.info.get_image_size_with_most_features()
|
||||
target_num_frames = \
|
||||
self.info.get_num_frames_with_most_features(seq_len, mm_counts)
|
||||
target_width, target_height = self.info.get_image_size_with_most_features()
|
||||
target_num_frames = self.info.get_num_frames_with_most_features(
|
||||
seq_len, mm_counts
|
||||
)
|
||||
|
||||
image_overrides = mm_options.get("image") if mm_options else None
|
||||
video_overrides = mm_options.get("video") if mm_options else None
|
||||
|
||||
mm_data = {
|
||||
"image":
|
||||
self._get_dummy_images(width=target_width,
|
||||
height=target_height,
|
||||
num_images=num_images,
|
||||
overrides=image_overrides),
|
||||
"video":
|
||||
self._get_dummy_videos(
|
||||
"image": self._get_dummy_images(
|
||||
width=target_width,
|
||||
height=target_height,
|
||||
num_images=num_images,
|
||||
overrides=image_overrides,
|
||||
),
|
||||
"video": self._get_dummy_videos(
|
||||
width=target_width,
|
||||
height=target_height,
|
||||
num_frames=target_num_frames,
|
||||
num_videos=num_videos,
|
||||
overrides=video_overrides,
|
||||
)
|
||||
),
|
||||
}
|
||||
return mm_data
|
||||
|
||||
|
||||
class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]
|
||||
):
|
||||
|
||||
class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]):
|
||||
def visual_indicators_to_visual_tokens(
|
||||
self,
|
||||
visual_indicators: list[int],
|
||||
) -> list[int]:
|
||||
"""
|
||||
Filter image indicators placeholders and convert them to corresponding
|
||||
Filter image indicators placeholders and convert them to corresponding
|
||||
tokens in visual tokenizer.
|
||||
"""
|
||||
hf_config = self.info.get_hf_config()
|
||||
vte_vocab_size = hf_config.visual_vocab_size
|
||||
return [
|
||||
vte_vocab_size - len(INDICATOR_IDS) + abs(x + 300) - 1
|
||||
for x in visual_indicators if x < -300
|
||||
for x in visual_indicators
|
||||
if x < -300
|
||||
]
|
||||
|
||||
def _call_hf_processor(
|
||||
@@ -388,7 +398,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]
|
||||
self,
|
||||
prompt_tokens: list[int],
|
||||
) -> list[int]:
|
||||
|
||||
return prompt_tokens
|
||||
|
||||
def _get_mm_fields_config(
|
||||
@@ -404,7 +413,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> list[PromptReplacement]:
|
||||
|
||||
def get_replacement_ovis(item_idx, modality: str):
|
||||
if modality == "image":
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
@@ -413,22 +421,26 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]
|
||||
out_item = out_mm_kwargs["video"][item_idx]
|
||||
grid = out_item["video_grids"].data
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
return hf_processor.construct_visual_placeholders(grid[0], )
|
||||
return hf_processor.construct_visual_placeholders(
|
||||
grid[0],
|
||||
)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=IMAGE_TOKEN if modality == "image" else VIDEO_TOKEN,
|
||||
replacement=partial(get_replacement_ovis, modality=modality),
|
||||
) for modality in ("image", "video")
|
||||
)
|
||||
for modality in ("image", "video")
|
||||
]
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(Ovis2_5MultiModalProcessor,
|
||||
info=Ovis2_5ProcessingInfo,
|
||||
dummy_inputs=Ovis2_5DummyInputsBuilder)
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Ovis2_5MultiModalProcessor,
|
||||
info=Ovis2_5ProcessingInfo,
|
||||
dummy_inputs=Ovis2_5DummyInputsBuilder,
|
||||
)
|
||||
class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
@@ -447,17 +459,18 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
prefix=f"{prefix}.visual_tokenizer",
|
||||
)
|
||||
|
||||
self.vte = VisualEmbedding(config.visual_vocab_size,
|
||||
config.hidden_size)
|
||||
self.vte = VisualEmbedding(config.visual_vocab_size, config.hidden_size)
|
||||
|
||||
text_model_type = self.config.get_text_config().model_type
|
||||
self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type]
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.get_language_model().make_empty_intermediate_tensors)
|
||||
self.get_language_model().make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object) -> Optional[OvisImagePatchInputs]:
|
||||
self, **kwargs: object
|
||||
) -> Optional[OvisImagePatchInputs]:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
indicator_tokens = kwargs.pop("indicator_tokens", None)
|
||||
grids = kwargs.pop("grids", None)
|
||||
@@ -466,12 +479,15 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
|
||||
if pixel_values is not None and indicator_tokens is not None:
|
||||
if not isinstance(pixel_values, (torch.Tensor, list)):
|
||||
raise ValueError("Incorrect type of pixel values. "
|
||||
f"Got type: {type(pixel_values)}")
|
||||
raise ValueError(
|
||||
f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
|
||||
)
|
||||
|
||||
if not isinstance(indicator_tokens, (torch.Tensor, list)):
|
||||
raise ValueError("Incorrect type of indicator_tokens. "
|
||||
f"Got type: {type(indicator_tokens)}")
|
||||
raise ValueError(
|
||||
"Incorrect type of indicator_tokens. "
|
||||
f"Got type: {type(indicator_tokens)}"
|
||||
)
|
||||
|
||||
return OvisImagePatchInputs(
|
||||
type="image_patches",
|
||||
@@ -480,15 +496,15 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
x.shape[0] // (self.config.vit_config.hidden_stride**2)
|
||||
for x in flatten_bn(pixel_values)
|
||||
],
|
||||
indicator_tokens=flatten_bn(flatten_bn(indicator_tokens),
|
||||
concat=True),
|
||||
indicator_tokens=flatten_bn(flatten_bn(indicator_tokens), concat=True),
|
||||
grids=flatten_bn(flatten_bn(grids), concat=True),
|
||||
)
|
||||
|
||||
raise AssertionError("This line should be unreachable.")
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object) -> Optional[OvisImagePatchInputs]:
|
||||
self, **kwargs: object
|
||||
) -> Optional[OvisImagePatchInputs]:
|
||||
pixel_values = kwargs.pop("video_pixel_values", None)
|
||||
indicator_tokens = kwargs.pop("video_indicator_tokens", None)
|
||||
grids = kwargs.pop("video_grids", None)
|
||||
@@ -497,12 +513,15 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
|
||||
if pixel_values is not None and indicator_tokens is not None:
|
||||
if not isinstance(pixel_values, (torch.Tensor, list)):
|
||||
raise ValueError("Incorrect type of pixel values. "
|
||||
f"Got type: {type(pixel_values)}")
|
||||
raise ValueError(
|
||||
f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
|
||||
)
|
||||
|
||||
if not isinstance(indicator_tokens, (torch.Tensor, list)):
|
||||
raise ValueError("Incorrect type of indicator_tokens. "
|
||||
f"Got type: {type(indicator_tokens)}")
|
||||
raise ValueError(
|
||||
"Incorrect type of indicator_tokens. "
|
||||
f"Got type: {type(indicator_tokens)}"
|
||||
)
|
||||
|
||||
return OvisVideoPatchInputs(
|
||||
type="video_patches",
|
||||
@@ -511,8 +530,7 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
x.shape[0] // (self.config.vit_config.hidden_stride**2)
|
||||
for x in flatten_bn(pixel_values)
|
||||
],
|
||||
indicator_tokens=flatten_bn(flatten_bn(indicator_tokens),
|
||||
concat=True),
|
||||
indicator_tokens=flatten_bn(flatten_bn(indicator_tokens), concat=True),
|
||||
grids=flatten_bn(flatten_bn(grids), concat=True),
|
||||
)
|
||||
|
||||
@@ -527,30 +545,32 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
grid_thws = image_input["grids"]
|
||||
|
||||
indicator_per_image = list(
|
||||
map(lambda x: 2 if x > 1 else x + 2, patches_per_image))
|
||||
map(lambda x: 2 if x > 1 else x + 2, patches_per_image)
|
||||
)
|
||||
|
||||
target_dtype = self.visual_tokenizer.dtype
|
||||
visual_tokens = self.visual_tokenizer(
|
||||
image_patches_flat.to(target_dtype), grid_thws)
|
||||
image_patches_flat.to(target_dtype), grid_thws
|
||||
)
|
||||
|
||||
visual_embeds = self.vte(visual_tokens) # 1:1 numeric eq.
|
||||
indicator_embeds = self.vte(indicator_tokens)
|
||||
|
||||
visual_embeds_per_image = visual_embeds.split(patches_per_image, dim=0)
|
||||
indicator_embeds_per_image = indicator_embeds.split(
|
||||
indicator_per_image)
|
||||
indicator_embeds_per_image = indicator_embeds.split(indicator_per_image)
|
||||
|
||||
vision_embeddings = []
|
||||
for indicator, visual in zip(indicator_embeds_per_image,
|
||||
visual_embeds_per_image):
|
||||
for indicator, visual in zip(
|
||||
indicator_embeds_per_image, visual_embeds_per_image
|
||||
):
|
||||
vision_embeddings_per_image = []
|
||||
visual = visual.unsqueeze(0)
|
||||
for i in range(visual.shape[0]):
|
||||
vision_embeddings_per_image.append(
|
||||
torch.cat([indicator[i:i + 1], visual[i]], dim=0))
|
||||
vision_embeddings_per_image.append(indicator[i + 1:])
|
||||
vision_embeddings.append(
|
||||
torch.cat(vision_embeddings_per_image, dim=0))
|
||||
torch.cat([indicator[i : i + 1], visual[i]], dim=0)
|
||||
)
|
||||
vision_embeddings_per_image.append(indicator[i + 1 :])
|
||||
vision_embeddings.append(torch.cat(vision_embeddings_per_image, dim=0))
|
||||
return tuple(vision_embeddings)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
@@ -559,20 +579,21 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
for input_key in kwargs:
|
||||
if input_key in ("pixel_values", "indicator_tokens",
|
||||
"grids") and "images" not in modalities:
|
||||
modalities["images"] = self._parse_and_validate_image_input(
|
||||
**kwargs)
|
||||
if input_key in ("video_pixel_values", "video_indicator_tokens",
|
||||
"video_grids") and "videos" not in modalities:
|
||||
modalities["videos"] = self._parse_and_validate_video_input(
|
||||
**kwargs)
|
||||
if (
|
||||
input_key in ("pixel_values", "indicator_tokens", "grids")
|
||||
and "images" not in modalities
|
||||
):
|
||||
modalities["images"] = self._parse_and_validate_image_input(**kwargs)
|
||||
if (
|
||||
input_key
|
||||
in ("video_pixel_values", "video_indicator_tokens", "video_grids")
|
||||
and "videos" not in modalities
|
||||
):
|
||||
modalities["videos"] = self._parse_and_validate_video_input(**kwargs)
|
||||
|
||||
return modalities
|
||||
|
||||
def get_multimodal_embeddings(self,
|
||||
**kwargs: object) -> MultiModalEmbeddings:
|
||||
|
||||
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
if not modalities:
|
||||
return []
|
||||
@@ -620,8 +641,7 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
logits = self.llm.compute_logits(hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str,
|
||||
torch.Tensor]]) -> set[str]:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user