diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index da472a164d..8c7cf1a68e 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -175,6 +175,24 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { }, marks=[pytest.mark.core_model], ), + "minimax_m3": VitCudagraphTestConfig( + model="MiniMaxAI/MiniMax-M3", + modalities=["image"], + image_prompt=("]~b]user\n]<]image[>[What is in this image?[e~[\n]~b]ai\n"), + compilation_config_overrides={ + "encoder_cudagraph_token_budgets": [512], + }, + vllm_runner_kwargs={ + "trust_remote_code": True, + "load_format": "dummy", + "hf_overrides": partial( + dummy_hf_overrides, + model_arch="MiniMaxM3SparseForConditionalGeneration", + ), + "mm_processor_kwargs": {"max_long_side_pixel": 336}, + }, + marks=[pytest.mark.core_model], + ), "qwen3_vl": VitCudagraphTestConfig( model="Qwen/Qwen3-VL-2B-Instruct", image_prompt=qwen_vl_chat_template( diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 4871e66a9a..40daa4a6ee 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -106,6 +106,9 @@ from vllm.models.minimax_m3.amd.sparse_attention_msa import ( MiniMaxM3SparseAiterPAImpl, MiniMaxM3SparseAiterPAPrefillMetadata, ) +from vllm.models.minimax_m3.common.encoder_cudagraph import ( + MiniMaxM3EncoderCudaGraphMixin, +) from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer from vllm.models.minimax_m3.common.mm_preprocess import ( MiniMaxM3VLDummyInputsBuilder, @@ -1685,7 +1688,11 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3): dummy_inputs=MiniMaxM3VLDummyInputsBuilder, ) class MiniMaxM3SparseForConditionalGeneration( - nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3 + nn.Module, + SupportsMultiModal, + MiniMaxM3EncoderCudaGraphMixin, + SupportsPP, + SupportsEagle3, ): """Top-level (VL) entry point for MiniMax M3. diff --git a/vllm/models/minimax_m3/common/encoder_cudagraph.py b/vllm/models/minimax_m3/common/encoder_cudagraph.py new file mode 100644 index 0000000000..7c55cde946 --- /dev/null +++ b/vllm/models/minimax_m3/common/encoder_cudagraph.py @@ -0,0 +1,281 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Encoder CUDA graph support for the MiniMax M3 vision tower. + +Mixin implementing the SupportsEncoderCudaGraph protocol for any host class +that holds the vision tower as ``self.vision_tower`` (a +``MiniMaxVLVisionModel``). The manager routes image items through this mixin; +video items use the eager encoder path. +""" + +from collections.abc import Hashable +from typing import TYPE_CHECKING, Any + +import torch + +from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.config.multimodal import MultiModalConfig + from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphCaptureInputs, + EncoderCudaGraphConfig, + EncoderCudaGraphReplayBuffers, + EncoderItemSpec, + ) + + +class MiniMaxM3EncoderCudaGraphMixin(SupportsEncoderCudaGraph): + """SupportsEncoderCudaGraph for hosts with ``self.vision_tower``.""" + + vision_tower: "MiniMaxVLVisionModel" + multimodal_config: "MultiModalConfig | None" + _encoder_cg_pad_totals: dict[int, int] = {} + + def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig": + from vllm.v1.attention.backends.registry import AttentionBackendEnum + from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig + + if ( + self.multimodal_config is not None + and self.multimodal_config.get_limit_per_prompt("image") == 0 + ): + raise ValueError( + "cudagraph_mm_encoder requires the MiniMax M3 vision tower, " + "but the image limit is 0. Disable cudagraph_mm_encoder." + ) + + mm_cfg = self.multimodal_config + if ( + mm_cfg is not None + and mm_cfg.mm_encoder_attn_dtype == "fp8" + and mm_cfg.mm_encoder_fp8_scale_path is None + ): + raise ValueError( + "cudagraph_mm_encoder requires static FP8 scales for the " + "MiniMax M3 vision encoder. Set --mm-encoder-fp8-scale-path " + "or disable cudagraph_mm_encoder." + ) + + # FlashInfer reads max_seqlen on the host and TORCH_SDPA calls + # .tolist() on CUDA cu_seqlens; neither can be captured. + backend = self.vision_tower.vision_model.attn_backend + if backend not in ( + AttentionBackendEnum.FLASH_ATTN, + AttentionBackendEnum.ROCM_AITER_FA, + ): + raise ValueError( + f"cudagraph_mm_encoder is not supported with the " + f"{backend.name} ViT attention backend for MiniMax M3. Set " + "--mm-encoder-attn-backend FLASH_ATTN or ROCM_AITER_FA " + "(ROCm), or disable cudagraph_mm_encoder." + ) + + pad_totals = self._encoder_cg_pad_totals + + def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None: + # Varlen attention requires cu_seqlens[-1] to equal the number of + # rows actually passed in. The captured buffers are sized for the + # full token budget, so a smaller real batch has to be completed + # with one trailing padding sequence; declaring fewer rows than the + # buffer holds is undefined behaviour and returns NaN on FlashAttn. + total = pad_totals.get(dst.data_ptr()) + if total is None: + raise RuntimeError("cu_seqlens replay buffer was not registered") + n = src.shape[0] + dst[:n].copy_(src[:n]) + dst[n:] = total + + def pad_rope_planes(dst: torch.Tensor, src: torch.Tensor) -> None: + # cos/sin planes are (3, N, half_rot_dim): the row axis is dim 1, + # not dim 0 as the manager's default slice-copy assumes. + dst.zero_() + dst[:, : src.shape[1]].copy_(src) + + return EncoderCudaGraphConfig( + modalities=["image"], + buffer_keys=[ + "pixel_values", + "rotary_cos", + "rotary_sin", + "cu_seqlens", + "max_seqlen", + ], + out_hidden_size=self.vision_tower.out_hidden_size, + padding_logics={ + "cu_seqlens": pad_cu_seqlens, + "rotary_cos": pad_rope_planes, + "rotary_sin": pad_rope_planes, + }, + ) + + def get_encoder_cudagraph_budget_range( + self, + vllm_config: "VllmConfig", + ) -> tuple[int, int]: + min_budget = 64 + max_budget = min( + vllm_config.scheduler_config.max_num_batched_tokens, + vllm_config.model_config.max_model_len, + ) + return (min_budget, max_budget) + + @staticmethod + def _get_grid_thw_list(mm_kwargs: dict[str, Any]) -> list[list[int]]: + grid_thw = mm_kwargs["image_grid_thw"] + if isinstance(grid_thw, torch.Tensor): + return [[int(x) for x in row] for row in grid_thw.tolist()] + return [[int(x) for x in row] for row in grid_thw] + + @staticmethod + def _get_pixel_values(mm_kwargs: dict[str, Any]) -> torch.Tensor: + pixel_values = mm_kwargs["pixel_values"] + if isinstance(pixel_values, list): + pixel_values = torch.cat(pixel_values) + return pixel_values + + def get_encoder_cudagraph_item_specs( + self, mm_kwargs: dict[str, Any] + ) -> list["EncoderItemSpec"]: + from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec + + merge = self.vision_tower.spatial_merge_size + return [ + EncoderItemSpec( + input_size=t * h * w, + output_tokens=t * h * w // (merge * merge), + ) + for t, h, w in self._get_grid_thw_list(mm_kwargs) + ] + + def select_encoder_cudagraph_items( + self, mm_kwargs: dict[str, Any], indices: list[int] + ) -> dict[str, Any]: + grid_thw_list = self._get_grid_thw_list(mm_kwargs) + pixel_values = self._get_pixel_values(mm_kwargs) + source_grid = mm_kwargs["image_grid_thw"] + + if len(indices) == 0: + empty_grid = ( + source_grid[:0] + if isinstance(source_grid, torch.Tensor) + else pixel_values.new_zeros((0, 3), dtype=torch.long) + ) + return {"pixel_values": pixel_values[:0], "image_grid_thw": empty_grid} + + patch_counts = [t * h * w for t, h, w in grid_thw_list] + cum = [0] + for pc in patch_counts: + cum.append(cum[-1] + pc) + selected_pv = torch.cat( + [pixel_values[cum[i] : cum[i + 1]] for i in indices], dim=0 + ) + grid_device = ( + source_grid.device if isinstance(source_grid, torch.Tensor) else None + ) + selected_grid = torch.tensor( + [grid_thw_list[i] for i in indices], + dtype=torch.long, + device=grid_device, + ) + return {"pixel_values": selected_pv, "image_grid_thw": selected_grid} + + def prepare_encoder_cudagraph_capture_inputs( + self, + token_budget: int, + max_batch_size: int, + max_frames_per_batch: int, + device: torch.device, + dtype: torch.dtype, + path: str = "default", + axis_keys: tuple[Hashable, ...] | None = None, + ) -> "EncoderCudaGraphCaptureInputs": + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphCaptureInputs, + ) + + merge = self.vision_tower.spatial_merge_size + # Output tokens per item in the dummy grid (ceiling so total >= budget). + per_item_out = (token_budget + max_batch_size - 1) // max_batch_size + grid_thw_list = [ + [1, merge, per_item_out * merge] for _ in range(max_batch_size) + ] + + embeddings = self.vision_tower.vision_model.embeddings + patch_dim = ( + embeddings.num_channels + * embeddings.temporal_patch_size + * embeddings.patch_size**2 + ) + total_patches = sum(t * h * w for t, h, w in grid_thw_list) + # Match the tower dtype exactly: MiniMaxVLPatchEmbed.forward mutates + # the module on dtype mismatch, which must not happen during capture. + dummy_pixel_values = torch.zeros( + total_patches, + patch_dim, + device=device, + dtype=self.vision_tower.dtype, + ) + + # The padding sequence can contain nearly all captured rows when a + # small image replays this graph. Leave one spare cu_seqlens slot. + metadata = self.vision_tower.vision_model.prepare_encoder_metadata( + grid_thw_list, + device=device, + max_batch_size=max_batch_size + 1, + max_seqlen_override=total_patches, + ) + + values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values} + values.update({k: v for k, v in metadata.items() if v is not None}) + self._encoder_cg_pad_totals[values["cu_seqlens"].data_ptr()] = total_patches + return EncoderCudaGraphCaptureInputs(values=values) + + def prepare_encoder_cudagraph_replay_buffers( + self, + mm_kwargs: dict[str, Any], + max_batch_size: int, + max_frames_per_batch: int, + path: str = "default", + ) -> "EncoderCudaGraphReplayBuffers": + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphReplayBuffers, + ) + + pixel_values = self._get_pixel_values(mm_kwargs) + # Unpadded: pad_cu_seqlens completes the tail with the padding sequence + # so cu_seqlens[-1] matches the captured buffer's row count. + metadata = self.vision_tower.vision_model.prepare_encoder_metadata( + self._get_grid_thw_list(mm_kwargs), + device=pixel_values.device, + ) + # FlashAttention reads this CPU scalar during capture, so the captured + # launch must retain the bound for the padded buffer. + metadata.pop("max_seqlen") + + values: dict[str, torch.Tensor | None] = {"pixel_values": pixel_values} + values.update(metadata) + return EncoderCudaGraphReplayBuffers(values=values) + + def encoder_cudagraph_forward( + self, + inputs: dict[str, torch.Tensor], + path: str = "default", + ) -> torch.Tensor: + pixel_values = inputs.pop("pixel_values") + # Remaining keys are consumed as encoder_metadata. + return self.vision_tower(pixel_values, grid_thw=None, encoder_metadata=inputs) + + def encoder_eager_forward( + self, + mm_kwargs: dict[str, Any], + path: str = "default", + ) -> torch.Tensor: + return self.vision_tower( + pixel_values=self._get_pixel_values(mm_kwargs).type( + self.vision_tower.dtype + ), + grid_thw=self._get_grid_thw_list(mm_kwargs), + ) diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py index ce56a179d0..fd9cff2c6d 100644 --- a/vllm/models/minimax_m3/common/vision_tower.py +++ b/vllm/models/minimax_m3/common/vision_tower.py @@ -392,19 +392,33 @@ class MiniMaxVLVisionTransformer(nn.Module): out.append([min(max_f, t - i), h, w]) return out - # ── Forward ────────────────────────────────────────────────────────── + # ── Encoder metadata (shared by eager forward and CUDA graph replay) ── - def forward( + def prepare_encoder_metadata( self, - pixel_values: torch.Tensor, grid_thw: list[list[int]], - ) -> torch.Tensor: - # pixel_values: (total_N, C * temporal_patch_size * patch_size²) - # Output: (total_N, hidden_size) + *, + device: torch.device, + max_batch_size: int | None = None, + max_seqlen_override: int | None = None, + ) -> dict[str, torch.Tensor | None]: + """Build all grid-dependent forward inputs as standalone tensors. - hidden = self.embeddings(pixel_values) # (total_N, hidden_size) - hidden = self.pre_layrnorm(hidden) + The eager path calls this per forward; encoder CUDA graphs call it at + capture (dummy grids, padded to fixed buffer sizes) and before each + replay (real grids, unpadded — the manager's padding logic completes + the captured buffers). + Args: + grid_thw: per-item [t, h, w] patch grids (host metadata). + device: device for the attention metadata tensors. + max_batch_size: pad cu_seqlens to this many segments with empty + (zero-length) trailing segments; None leaves it unpadded. + max_seqlen_override: bake this max_seqlen instead of deriving it + from the grids; CUDA graph capture passes a worst-case value + because max_seqlen is a CPU scalar baked into the graph. + + """ limited = self._apply_max_frames_limit(grid_thw) # Token-level cumulative sequence lengths (one segment per limited grid). @@ -412,15 +426,23 @@ class MiniMaxVLVisionTransformer(nn.Module): cu_seqlens_np = np.zeros(len(lens) + 1, dtype=np.int32) np.cumsum(np.array(lens, dtype=np.int32), out=cu_seqlens_np[1:]) + if max_batch_size is not None and len(lens) < max_batch_size: + pad = np.full(max_batch_size - len(lens), cu_seqlens_np[-1], dtype=np.int32) + cu_seqlens_np = np.concatenate([cu_seqlens_np, pad]) + # Backend-specific encoder metadata. For FLASH_ATTN this returns the raw # token cu_seqlens, the max segment length, and sequence_lengths=None; # for FLASHINFER (cuDNN) it repacks cu_seqlens into element-offset # indptrs, buckets max_seqlen, and builds padded per-sequence lengths. sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens( - self.attn_backend, cu_seqlens_np, hidden.device + self.attn_backend, cu_seqlens_np, device ) max_seqlen = torch.tensor( - MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens_np), + max_seqlen_override + if max_seqlen_override is not None + else MMEncoderAttention.compute_max_seqlen( + self.attn_backend, cu_seqlens_np + ), dtype=torch.int32, ) cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens( @@ -428,7 +450,7 @@ class MiniMaxVLVisionTransformer(nn.Module): cu_seqlens_np, self.hidden_size, self.tp_size, - hidden.device, + device, ) # 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for @@ -442,6 +464,45 @@ class MiniMaxVLVisionTransformer(nn.Module): self.spatial_merge_size, ) + return { + "rotary_cos": rotary_cos, + "rotary_sin": rotary_sin, + "cu_seqlens": cu_seqlens, + "max_seqlen": max_seqlen, + "sequence_lengths": sequence_lengths, + } + + # ── Forward ────────────────────────────────────────────────────────── + + def forward( + self, + pixel_values: torch.Tensor, + grid_thw: list[list[int]] | None = None, + *, + encoder_metadata: dict[str, torch.Tensor | None] | None = None, + ) -> torch.Tensor: + # pixel_values: (total_N, C * temporal_patch_size * patch_size²) + # Output: (total_N, hidden_size) + + hidden = self.embeddings(pixel_values) # (total_N, hidden_size) + hidden = self.pre_layrnorm(hidden) + + if encoder_metadata is None: + assert grid_thw is not None + encoder_metadata = self.prepare_encoder_metadata( + grid_thw, device=hidden.device + ) + + cu_seqlens = encoder_metadata["cu_seqlens"] + rotary_cos = encoder_metadata["rotary_cos"] + rotary_sin = encoder_metadata["rotary_sin"] + max_seqlen = encoder_metadata["max_seqlen"] + sequence_lengths = encoder_metadata.get("sequence_lengths") + assert cu_seqlens is not None + assert rotary_cos is not None + assert rotary_sin is not None + assert max_seqlen is not None + # Encoder expects (N, 1, hidden_size) — add batch dim hidden = hidden.unsqueeze(1) @@ -600,9 +661,15 @@ class MiniMaxVLVisionModel(nn.Module): def forward( self, pixel_values: torch.Tensor, - grid_thw: list[list[int]], + grid_thw: list[list[int]] | None = None, + *, + encoder_metadata: dict[str, torch.Tensor | None] | None = None, ) -> torch.Tensor: - hidden = self.vision_model(pixel_values=pixel_values, grid_thw=grid_thw) + hidden = self.vision_model( + pixel_values=pixel_values, + grid_thw=grid_thw, + encoder_metadata=encoder_metadata, + ) if hidden.dim() == 3: hidden = hidden.squeeze(0) hidden = self.multi_modal_projector(hidden) diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index 4fb260330f..c16918bd37 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -70,6 +70,9 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model +from vllm.models.minimax_m3.common.encoder_cudagraph import ( + MiniMaxM3EncoderCudaGraphMixin, +) from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer from vllm.models.minimax_m3.common.mm_preprocess import ( MiniMaxM3VLDummyInputsBuilder, @@ -1076,7 +1079,11 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3): dummy_inputs=MiniMaxM3VLDummyInputsBuilder, ) class MiniMaxM3SparseForConditionalGeneration( - nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3 + nn.Module, + SupportsMultiModal, + MiniMaxM3EncoderCudaGraphMixin, + SupportsPP, + SupportsEagle3, ): """Top-level (VL) entry point for MiniMax M3.