Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+127 -107
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
""" PyTorch Ovis model."""
"""PyTorch Ovis model."""
from collections.abc import Iterable, Mapping
from functools import partial
from typing import Literal, Optional, TypedDict, Union
@@ -13,18 +14,26 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.ovis import (OvisImagePatchInputs,
VisualEmbedding)
from vllm.model_executor.models.ovis import OvisImagePatchInputs, VisualEmbedding
from vllm.model_executor.models.siglip2navit import Siglip2NavitModel
from vllm.model_executor.models.utils import (AutoWeightsLoader, flatten_bn,
init_vllm_registered_model,
maybe_prefix)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
flatten_bn,
init_vllm_registered_model,
maybe_prefix,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
MultiModalKwargsItems)
from vllm.multimodal.inputs import (
MultiModalDataDict,
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
from vllm.multimodal.processing import (BaseMultiModalProcessor,
BaseProcessingInfo, PromptReplacement)
from vllm.multimodal.processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
)
from vllm.multimodal.profiling import BaseDummyInputsBuilder
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor
@@ -71,12 +80,14 @@ class OvisVideoPatchInputs(TypedDict):
def _ovis2_5_field_config():
return dict(pixel_values=MultiModalFieldConfig.batched("image"),
grids=MultiModalFieldConfig.batched("image"),
indicator_tokens=MultiModalFieldConfig.batched("image"),
video_pixel_values=MultiModalFieldConfig.batched("video"),
video_indicator_tokens=MultiModalFieldConfig.batched("video"),
video_grids=MultiModalFieldConfig.batched("video"))
return dict(
pixel_values=MultiModalFieldConfig.batched("image"),
grids=MultiModalFieldConfig.batched("image"),
indicator_tokens=MultiModalFieldConfig.batched("image"),
video_pixel_values=MultiModalFieldConfig.batched("video"),
video_indicator_tokens=MultiModalFieldConfig.batched("video"),
video_grids=MultiModalFieldConfig.batched("video"),
)
class VisualTokenizer(torch.nn.Module):
@@ -108,7 +119,9 @@ class VisualTokenizer(torch.nn.Module):
head_dim,
bias=False,
return_bias=False,
), torch.nn.LayerNorm(head_dim))
),
torch.nn.LayerNorm(head_dim),
)
def _init_backbone(
self,
@@ -119,12 +132,13 @@ class VisualTokenizer(torch.nn.Module):
):
model_type = config.model_type
if model_type == "siglip2_navit":
return Siglip2NavitModel(config=config,
quant_config=quant_config,
prefix=prefix,
use_data_parallel=use_data_parallel)
raise ValueError(
f"Unsupported visual tokenizer model_type: {model_type}")
return Siglip2NavitModel(
config=config,
quant_config=quant_config,
prefix=prefix,
use_data_parallel=use_data_parallel,
)
raise ValueError(f"Unsupported visual tokenizer model_type: {model_type}")
@property
def dtype(self) -> torch.dtype:
@@ -135,22 +149,22 @@ class VisualTokenizer(torch.nn.Module):
return next(self.head.parameters()).device
def tokenize(self, logits: torch.Tensor) -> torch.Tensor:
tokens = torch.softmax(logits, dim=-1,
dtype=torch.float32).to(logits.dtype)
tokens = torch.softmax(logits, dim=-1, dtype=torch.float32).to(logits.dtype)
return tokens
def encode(self, pixel_values: torch.Tensor,
grid_thws: torch.Tensor) -> torch.Tensor:
def encode(
self, pixel_values: torch.Tensor, grid_thws: torch.Tensor
) -> torch.Tensor:
features = self.vit(pixel_values, grid_thws)
# refer to qwen2.5-vl patchmerger
seq_len, _ = features.shape
features = features.reshape(seq_len // (self.config.hidden_stride**2),
-1)
features = features.reshape(seq_len // (self.config.hidden_stride**2), -1)
return features
def forward(self, pixel_values: torch.Tensor,
grid_thws: torch.Tensor) -> torch.Tensor:
def forward(
self, pixel_values: torch.Tensor, grid_thws: torch.Tensor
) -> torch.Tensor:
features = self.encode(pixel_values, grid_thws)
logits = self.head(features)
tokens = self.tokenize(logits)
@@ -167,7 +181,6 @@ class VisualTokenizer(torch.nn.Module):
class Ovis2_5ProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config()
@@ -220,8 +233,9 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
def get_max_image_tokens(self) -> int:
target_width, target_height = self.get_image_size_with_most_features()
return self.get_num_image_tokens(image_width=target_width,
image_height=target_height)
return self.get_num_image_tokens(
image_width=target_width, image_height=target_height
)
def _get_max_video_frames(self, max_tokens: int) -> int:
target_width, target_height = self.get_image_size_with_most_features()
@@ -247,8 +261,7 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
max_images = mm_counts.get("image", 0)
max_videos = mm_counts.get("video", 0)
max_image_tokens = self.get_max_image_tokens() * max_images
max_total_frames = self._get_max_video_frames(seq_len -
max_image_tokens)
max_total_frames = self._get_max_video_frames(seq_len - max_image_tokens)
max_frames_per_video = max_total_frames // max(max_videos, 1)
return max(max_frames_per_video, 1)
@@ -260,9 +273,9 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
num_frames: int,
image_processor: Optional[BaseImageProcessor],
) -> int:
num_video_tokens = self.get_num_image_tokens(image_width=image_width,
image_height=image_height,
num_frames=num_frames)
num_video_tokens = self.get_num_image_tokens(
image_width=image_width, image_height=image_height, num_frames=num_frames
)
return num_video_tokens
def get_max_video_tokens(
@@ -274,14 +287,12 @@ class Ovis2_5ProcessingInfo(BaseProcessingInfo):
return self.get_num_video_tokens(
image_width=target_width,
image_height=target_height,
num_frames=self.get_num_frames_with_most_features(
seq_len, mm_counts),
num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
image_processor=None,
)
class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
@@ -296,48 +307,47 @@ class Ovis2_5DummyInputsBuilder(BaseDummyInputsBuilder[Ovis2_5ProcessingInfo]):
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
target_width, target_height = \
self.info.get_image_size_with_most_features()
target_num_frames = \
self.info.get_num_frames_with_most_features(seq_len, mm_counts)
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts
)
image_overrides = mm_options.get("image") if mm_options else None
video_overrides = mm_options.get("video") if mm_options else None
mm_data = {
"image":
self._get_dummy_images(width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides),
"video":
self._get_dummy_videos(
"image": self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
),
"video": self._get_dummy_videos(
width=target_width,
height=target_height,
num_frames=target_num_frames,
num_videos=num_videos,
overrides=video_overrides,
)
),
}
return mm_data
class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]
):
class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]):
def visual_indicators_to_visual_tokens(
self,
visual_indicators: list[int],
) -> list[int]:
"""
Filter image indicators placeholders and convert them to corresponding
Filter image indicators placeholders and convert them to corresponding
tokens in visual tokenizer.
"""
hf_config = self.info.get_hf_config()
vte_vocab_size = hf_config.visual_vocab_size
return [
vte_vocab_size - len(INDICATOR_IDS) + abs(x + 300) - 1
for x in visual_indicators if x < -300
for x in visual_indicators
if x < -300
]
def _call_hf_processor(
@@ -388,7 +398,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]
self,
prompt_tokens: list[int],
) -> list[int]:
return prompt_tokens
def _get_mm_fields_config(
@@ -404,7 +413,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> list[PromptReplacement]:
def get_replacement_ovis(item_idx, modality: str):
if modality == "image":
out_item = out_mm_kwargs["image"][item_idx]
@@ -413,22 +421,26 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo]
out_item = out_mm_kwargs["video"][item_idx]
grid = out_item["video_grids"].data
hf_processor = self.info.get_hf_processor()
return hf_processor.construct_visual_placeholders(grid[0], )
return hf_processor.construct_visual_placeholders(
grid[0],
)
return [
PromptReplacement(
modality=modality,
target=IMAGE_TOKEN if modality == "image" else VIDEO_TOKEN,
replacement=partial(get_replacement_ovis, modality=modality),
) for modality in ("image", "video")
)
for modality in ("image", "video")
]
@MULTIMODAL_REGISTRY.register_processor(Ovis2_5MultiModalProcessor,
info=Ovis2_5ProcessingInfo,
dummy_inputs=Ovis2_5DummyInputsBuilder)
@MULTIMODAL_REGISTRY.register_processor(
Ovis2_5MultiModalProcessor,
info=Ovis2_5ProcessingInfo,
dummy_inputs=Ovis2_5DummyInputsBuilder,
)
class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
@@ -447,17 +459,18 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
prefix=f"{prefix}.visual_tokenizer",
)
self.vte = VisualEmbedding(config.visual_vocab_size,
config.hidden_size)
self.vte = VisualEmbedding(config.visual_vocab_size, config.hidden_size)
text_model_type = self.config.get_text_config().model_type
self.image_pad_token_id = IMAGE_PAD_TOKEN_ID_MAP[text_model_type]
self.make_empty_intermediate_tensors = (
self.get_language_model().make_empty_intermediate_tensors)
self.get_language_model().make_empty_intermediate_tensors
)
def _parse_and_validate_image_input(
self, **kwargs: object) -> Optional[OvisImagePatchInputs]:
self, **kwargs: object
) -> Optional[OvisImagePatchInputs]:
pixel_values = kwargs.pop("pixel_values", None)
indicator_tokens = kwargs.pop("indicator_tokens", None)
grids = kwargs.pop("grids", None)
@@ -466,12 +479,15 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
if pixel_values is not None and indicator_tokens is not None:
if not isinstance(pixel_values, (torch.Tensor, list)):
raise ValueError("Incorrect type of pixel values. "
f"Got type: {type(pixel_values)}")
raise ValueError(
f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
)
if not isinstance(indicator_tokens, (torch.Tensor, list)):
raise ValueError("Incorrect type of indicator_tokens. "
f"Got type: {type(indicator_tokens)}")
raise ValueError(
"Incorrect type of indicator_tokens. "
f"Got type: {type(indicator_tokens)}"
)
return OvisImagePatchInputs(
type="image_patches",
@@ -480,15 +496,15 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
x.shape[0] // (self.config.vit_config.hidden_stride**2)
for x in flatten_bn(pixel_values)
],
indicator_tokens=flatten_bn(flatten_bn(indicator_tokens),
concat=True),
indicator_tokens=flatten_bn(flatten_bn(indicator_tokens), concat=True),
grids=flatten_bn(flatten_bn(grids), concat=True),
)
raise AssertionError("This line should be unreachable.")
def _parse_and_validate_video_input(
self, **kwargs: object) -> Optional[OvisImagePatchInputs]:
self, **kwargs: object
) -> Optional[OvisImagePatchInputs]:
pixel_values = kwargs.pop("video_pixel_values", None)
indicator_tokens = kwargs.pop("video_indicator_tokens", None)
grids = kwargs.pop("video_grids", None)
@@ -497,12 +513,15 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
if pixel_values is not None and indicator_tokens is not None:
if not isinstance(pixel_values, (torch.Tensor, list)):
raise ValueError("Incorrect type of pixel values. "
f"Got type: {type(pixel_values)}")
raise ValueError(
f"Incorrect type of pixel values. Got type: {type(pixel_values)}"
)
if not isinstance(indicator_tokens, (torch.Tensor, list)):
raise ValueError("Incorrect type of indicator_tokens. "
f"Got type: {type(indicator_tokens)}")
raise ValueError(
"Incorrect type of indicator_tokens. "
f"Got type: {type(indicator_tokens)}"
)
return OvisVideoPatchInputs(
type="video_patches",
@@ -511,8 +530,7 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
x.shape[0] // (self.config.vit_config.hidden_stride**2)
for x in flatten_bn(pixel_values)
],
indicator_tokens=flatten_bn(flatten_bn(indicator_tokens),
concat=True),
indicator_tokens=flatten_bn(flatten_bn(indicator_tokens), concat=True),
grids=flatten_bn(flatten_bn(grids), concat=True),
)
@@ -527,30 +545,32 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
grid_thws = image_input["grids"]
indicator_per_image = list(
map(lambda x: 2 if x > 1 else x + 2, patches_per_image))
map(lambda x: 2 if x > 1 else x + 2, patches_per_image)
)
target_dtype = self.visual_tokenizer.dtype
visual_tokens = self.visual_tokenizer(
image_patches_flat.to(target_dtype), grid_thws)
image_patches_flat.to(target_dtype), grid_thws
)
visual_embeds = self.vte(visual_tokens) # 1:1 numeric eq.
indicator_embeds = self.vte(indicator_tokens)
visual_embeds_per_image = visual_embeds.split(patches_per_image, dim=0)
indicator_embeds_per_image = indicator_embeds.split(
indicator_per_image)
indicator_embeds_per_image = indicator_embeds.split(indicator_per_image)
vision_embeddings = []
for indicator, visual in zip(indicator_embeds_per_image,
visual_embeds_per_image):
for indicator, visual in zip(
indicator_embeds_per_image, visual_embeds_per_image
):
vision_embeddings_per_image = []
visual = visual.unsqueeze(0)
for i in range(visual.shape[0]):
vision_embeddings_per_image.append(
torch.cat([indicator[i:i + 1], visual[i]], dim=0))
vision_embeddings_per_image.append(indicator[i + 1:])
vision_embeddings.append(
torch.cat(vision_embeddings_per_image, dim=0))
torch.cat([indicator[i : i + 1], visual[i]], dim=0)
)
vision_embeddings_per_image.append(indicator[i + 1 :])
vision_embeddings.append(torch.cat(vision_embeddings_per_image, dim=0))
return tuple(vision_embeddings)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
@@ -559,20 +579,21 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
for input_key in kwargs:
if input_key in ("pixel_values", "indicator_tokens",
"grids") and "images" not in modalities:
modalities["images"] = self._parse_and_validate_image_input(
**kwargs)
if input_key in ("video_pixel_values", "video_indicator_tokens",
"video_grids") and "videos" not in modalities:
modalities["videos"] = self._parse_and_validate_video_input(
**kwargs)
if (
input_key in ("pixel_values", "indicator_tokens", "grids")
and "images" not in modalities
):
modalities["images"] = self._parse_and_validate_image_input(**kwargs)
if (
input_key
in ("video_pixel_values", "video_indicator_tokens", "video_grids")
and "videos" not in modalities
):
modalities["videos"] = self._parse_and_validate_video_input(**kwargs)
return modalities
def get_multimodal_embeddings(self,
**kwargs: object) -> MultiModalEmbeddings:
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
if not modalities:
return []
@@ -620,8 +641,7 @@ class Ovis2_5(nn.Module, SupportsMultiModal, SupportsPP):
logits = self.llm.compute_logits(hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str,
torch.Tensor]]) -> set[str]:
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)