[MM][Perf][CG] Support ViT full cudagraphs for mllama4 (#40660)
Signed-off-by: allgather <all2allops@gmail.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -19,7 +19,7 @@
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping
|
||||
from itertools import tee
|
||||
from typing import Annotated, Literal
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -78,6 +78,7 @@ from .interfaces import (
|
||||
MixtureOfExperts,
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
@@ -105,7 +106,7 @@ class Llama4ImagePatchInputs(TensorSchema):
|
||||
|
||||
patches_per_image: Annotated[torch.Tensor, TensorShape("batch_size")]
|
||||
"""
|
||||
The number of total patches for each image in the batch.
|
||||
The number of chunked image tiles for each image in the batch.
|
||||
|
||||
This is used to split the embeddings which has the first two dimensions
|
||||
flattened just like `pixel_values`.
|
||||
@@ -731,6 +732,7 @@ class Llama4ForConditionalGeneration(
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
MixtureOfExperts,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
):
|
||||
@@ -828,10 +830,161 @@ class Llama4ForConditionalGeneration(
|
||||
num_physical_experts, num_local_physical_experts
|
||||
)
|
||||
|
||||
def get_image_patches_per_chunk(self) -> int:
|
||||
return Mllama4ProcessingInfo.get_patch_per_chunk(self.config.vision_config)
|
||||
|
||||
def encode_image_chunks(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
*,
|
||||
use_data_parallel: bool,
|
||||
) -> torch.Tensor:
|
||||
if use_data_parallel:
|
||||
vision_embeddings = run_dp_sharded_vision_model(
|
||||
pixel_values, self.vision_model
|
||||
)
|
||||
else:
|
||||
vision_embeddings = self.vision_model(pixel_values)
|
||||
|
||||
return self.multi_modal_projector(vision_embeddings)
|
||||
|
||||
def get_encoder_cudagraph_config(self):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphConfig,
|
||||
)
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
buffer_keys=["pixel_values"],
|
||||
out_hidden_size=self.config.text_config.hidden_size,
|
||||
)
|
||||
|
||||
def get_input_modality(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> str:
|
||||
return "image"
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[int, int]:
|
||||
min_budget = self.get_image_patches_per_chunk()
|
||||
max_budget = min(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
self.vllm_config.model_config.max_model_len,
|
||||
)
|
||||
return (min_budget, max_budget)
|
||||
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
|
||||
|
||||
patches_per_chunk = self.get_image_patches_per_chunk()
|
||||
return [
|
||||
EncoderItemSpec(
|
||||
input_size=num_chunks,
|
||||
output_tokens=num_chunks * patches_per_chunk,
|
||||
)
|
||||
for num_chunks in mm_kwargs["patches_per_image"].tolist()
|
||||
]
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
indices: list[int],
|
||||
) -> dict[str, Any]:
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
patches_per_image = mm_kwargs["patches_per_image"]
|
||||
|
||||
if len(indices) == 0:
|
||||
return {
|
||||
"pixel_values": pixel_values[:0],
|
||||
"patches_per_image": patches_per_image[:0],
|
||||
}
|
||||
|
||||
cum_chunks = [0]
|
||||
for num_chunks in patches_per_image.tolist():
|
||||
cum_chunks.append(cum_chunks[-1] + num_chunks)
|
||||
|
||||
selected_pixel_values = torch.cat(
|
||||
[pixel_values[cum_chunks[i] : cum_chunks[i + 1]] for i in indices],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
return {
|
||||
"pixel_values": selected_pixel_values,
|
||||
"patches_per_image": patches_per_image[indices],
|
||||
}
|
||||
|
||||
def prepare_encoder_cudagraph_capture_inputs(
|
||||
self,
|
||||
token_budget: int,
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
)
|
||||
|
||||
vision_config = self.config.vision_config
|
||||
patches_per_chunk = self.get_image_patches_per_chunk()
|
||||
chunks_per_capture = max(
|
||||
1, (token_budget + patches_per_chunk - 1) // patches_per_chunk
|
||||
)
|
||||
dummy_pixel_values = torch.randn(
|
||||
chunks_per_capture,
|
||||
vision_config.num_channels,
|
||||
vision_config.image_size,
|
||||
vision_config.image_size,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
return EncoderCudaGraphCaptureInputs(
|
||||
values={"pixel_values": dummy_pixel_values},
|
||||
)
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphReplayBuffers,
|
||||
)
|
||||
|
||||
return EncoderCudaGraphReplayBuffers(
|
||||
values={"pixel_values": mm_kwargs["pixel_values"]},
|
||||
)
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self,
|
||||
inputs: dict[str, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
return self.encode_image_chunks(
|
||||
inputs["pixel_values"],
|
||||
use_data_parallel=False,
|
||||
).flatten(0, 1)
|
||||
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> torch.Tensor:
|
||||
return self.encode_image_chunks(
|
||||
mm_kwargs["pixel_values"],
|
||||
use_data_parallel=False,
|
||||
).flatten(0, 1)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> Llama4ImagePatchInputs | None:
|
||||
# num_images, 1, num_chunks, channel, image_size, image_size
|
||||
# total_num_chunks, channel, image_size, image_size
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
if pixel_values is None:
|
||||
return None
|
||||
@@ -853,15 +1006,10 @@ class Llama4ForConditionalGeneration(
|
||||
pixel_values = image_input["pixel_values"]
|
||||
patches_per_image = image_input["patches_per_image"].tolist()
|
||||
|
||||
# shard image input
|
||||
if self.use_data_parallel:
|
||||
vision_embeddings_flat = run_dp_sharded_vision_model(
|
||||
pixel_values, self.vision_model
|
||||
)
|
||||
else:
|
||||
vision_embeddings_flat = self.vision_model(pixel_values)
|
||||
|
||||
vision_embeddings_flat = self.multi_modal_projector(vision_embeddings_flat)
|
||||
vision_embeddings_flat = self.encode_image_chunks(
|
||||
pixel_values,
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
)
|
||||
|
||||
return [
|
||||
img.flatten(0, 1)
|
||||
|
||||
Reference in New Issue
Block a user