[Minimax-M3] Add Encoder CUDA graph support (#58673)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Thien Tran
2026-09-28 04:17:36 +00:00
committed by GitHub
co-authored by Codex
parent 6f65fde419
commit 55de40a2fc
5 changed files with 395 additions and 15 deletions
@@ -175,6 +175,24 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
},
marks=[pytest.mark.core_model],
),
"minimax_m3": VitCudagraphTestConfig(
model="MiniMaxAI/MiniMax-M3",
modalities=["image"],
image_prompt=("]~b]user\n]<]image[>[What is in this image?[e~[\n]~b]ai\n"),
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [512],
},
vllm_runner_kwargs={
"trust_remote_code": True,
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="MiniMaxM3SparseForConditionalGeneration",
),
"mm_processor_kwargs": {"max_long_side_pixel": 336},
},
marks=[pytest.mark.core_model],
),
"qwen3_vl": VitCudagraphTestConfig(
model="Qwen/Qwen3-VL-2B-Instruct",
image_prompt=qwen_vl_chat_template(
+8 -1
View File
@@ -106,6 +106,9 @@ from vllm.models.minimax_m3.amd.sparse_attention_msa import (
MiniMaxM3SparseAiterPAImpl,
MiniMaxM3SparseAiterPAPrefillMetadata,
)
from vllm.models.minimax_m3.common.encoder_cudagraph import (
MiniMaxM3EncoderCudaGraphMixin,
)
from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer
from vllm.models.minimax_m3.common.mm_preprocess import (
MiniMaxM3VLDummyInputsBuilder,
@@ -1685,7 +1688,11 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
dummy_inputs=MiniMaxM3VLDummyInputsBuilder,
)
class MiniMaxM3SparseForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3
nn.Module,
SupportsMultiModal,
MiniMaxM3EncoderCudaGraphMixin,
SupportsPP,
SupportsEagle3,
):
"""Top-level (VL) entry point for MiniMax M3.
@@ -0,0 +1,281 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encoder CUDA graph support for the MiniMax M3 vision tower.
Mixin implementing the SupportsEncoderCudaGraph protocol for any host class
that holds the vision tower as ``self.vision_tower`` (a
``MiniMaxVLVisionModel``). The manager routes image items through this mixin;
video items use the eager encoder path.
"""
from collections.abc import Hashable
from typing import TYPE_CHECKING, Any
import torch
from vllm.model_executor.models.interfaces import SupportsEncoderCudaGraph
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.models.minimax_m3.common.vision_tower import MiniMaxVLVisionModel
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
EncoderCudaGraphReplayBuffers,
EncoderItemSpec,
)
class MiniMaxM3EncoderCudaGraphMixin(SupportsEncoderCudaGraph):
"""SupportsEncoderCudaGraph for hosts with ``self.vision_tower``."""
vision_tower: "MiniMaxVLVisionModel"
multimodal_config: "MultiModalConfig | None"
_encoder_cg_pad_totals: dict[int, int] = {}
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig":
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
if (
self.multimodal_config is not None
and self.multimodal_config.get_limit_per_prompt("image") == 0
):
raise ValueError(
"cudagraph_mm_encoder requires the MiniMax M3 vision tower, "
"but the image limit is 0. Disable cudagraph_mm_encoder."
)
mm_cfg = self.multimodal_config
if (
mm_cfg is not None
and mm_cfg.mm_encoder_attn_dtype == "fp8"
and mm_cfg.mm_encoder_fp8_scale_path is None
):
raise ValueError(
"cudagraph_mm_encoder requires static FP8 scales for the "
"MiniMax M3 vision encoder. Set --mm-encoder-fp8-scale-path "
"or disable cudagraph_mm_encoder."
)
# FlashInfer reads max_seqlen on the host and TORCH_SDPA calls
# .tolist() on CUDA cu_seqlens; neither can be captured.
backend = self.vision_tower.vision_model.attn_backend
if backend not in (
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.ROCM_AITER_FA,
):
raise ValueError(
f"cudagraph_mm_encoder is not supported with the "
f"{backend.name} ViT attention backend for MiniMax M3. Set "
"--mm-encoder-attn-backend FLASH_ATTN or ROCM_AITER_FA "
"(ROCm), or disable cudagraph_mm_encoder."
)
pad_totals = self._encoder_cg_pad_totals
def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None:
# Varlen attention requires cu_seqlens[-1] to equal the number of
# rows actually passed in. The captured buffers are sized for the
# full token budget, so a smaller real batch has to be completed
# with one trailing padding sequence; declaring fewer rows than the
# buffer holds is undefined behaviour and returns NaN on FlashAttn.
total = pad_totals.get(dst.data_ptr())
if total is None:
raise RuntimeError("cu_seqlens replay buffer was not registered")
n = src.shape[0]
dst[:n].copy_(src[:n])
dst[n:] = total
def pad_rope_planes(dst: torch.Tensor, src: torch.Tensor) -> None:
# cos/sin planes are (3, N, half_rot_dim): the row axis is dim 1,
# not dim 0 as the manager's default slice-copy assumes.
dst.zero_()
dst[:, : src.shape[1]].copy_(src)
return EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=[
"pixel_values",
"rotary_cos",
"rotary_sin",
"cu_seqlens",
"max_seqlen",
],
out_hidden_size=self.vision_tower.out_hidden_size,
padding_logics={
"cu_seqlens": pad_cu_seqlens,
"rotary_cos": pad_rope_planes,
"rotary_sin": pad_rope_planes,
},
)
def get_encoder_cudagraph_budget_range(
self,
vllm_config: "VllmConfig",
) -> tuple[int, int]:
min_budget = 64
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
vllm_config.model_config.max_model_len,
)
return (min_budget, max_budget)
@staticmethod
def _get_grid_thw_list(mm_kwargs: dict[str, Any]) -> list[list[int]]:
grid_thw = mm_kwargs["image_grid_thw"]
if isinstance(grid_thw, torch.Tensor):
return [[int(x) for x in row] for row in grid_thw.tolist()]
return [[int(x) for x in row] for row in grid_thw]
@staticmethod
def _get_pixel_values(mm_kwargs: dict[str, Any]) -> torch.Tensor:
pixel_values = mm_kwargs["pixel_values"]
if isinstance(pixel_values, list):
pixel_values = torch.cat(pixel_values)
return pixel_values
def get_encoder_cudagraph_item_specs(
self, mm_kwargs: dict[str, Any]
) -> list["EncoderItemSpec"]:
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
merge = self.vision_tower.spatial_merge_size
return [
EncoderItemSpec(
input_size=t * h * w,
output_tokens=t * h * w // (merge * merge),
)
for t, h, w in self._get_grid_thw_list(mm_kwargs)
]
def select_encoder_cudagraph_items(
self, mm_kwargs: dict[str, Any], indices: list[int]
) -> dict[str, Any]:
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
pixel_values = self._get_pixel_values(mm_kwargs)
source_grid = mm_kwargs["image_grid_thw"]
if len(indices) == 0:
empty_grid = (
source_grid[:0]
if isinstance(source_grid, torch.Tensor)
else pixel_values.new_zeros((0, 3), dtype=torch.long)
)
return {"pixel_values": pixel_values[:0], "image_grid_thw": empty_grid}
patch_counts = [t * h * w for t, h, w in grid_thw_list]
cum = [0]
for pc in patch_counts:
cum.append(cum[-1] + pc)
selected_pv = torch.cat(
[pixel_values[cum[i] : cum[i + 1]] for i in indices], dim=0
)
grid_device = (
source_grid.device if isinstance(source_grid, torch.Tensor) else None
)
selected_grid = torch.tensor(
[grid_thw_list[i] for i in indices],
dtype=torch.long,
device=grid_device,
)
return {"pixel_values": selected_pv, "image_grid_thw": selected_grid}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
) -> "EncoderCudaGraphCaptureInputs":
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
)
merge = self.vision_tower.spatial_merge_size
# Output tokens per item in the dummy grid (ceiling so total >= budget).
per_item_out = (token_budget + max_batch_size - 1) // max_batch_size
grid_thw_list = [
[1, merge, per_item_out * merge] for _ in range(max_batch_size)
]
embeddings = self.vision_tower.vision_model.embeddings
patch_dim = (
embeddings.num_channels
* embeddings.temporal_patch_size
* embeddings.patch_size**2
)
total_patches = sum(t * h * w for t, h, w in grid_thw_list)
# Match the tower dtype exactly: MiniMaxVLPatchEmbed.forward mutates
# the module on dtype mismatch, which must not happen during capture.
dummy_pixel_values = torch.zeros(
total_patches,
patch_dim,
device=device,
dtype=self.vision_tower.dtype,
)
# The padding sequence can contain nearly all captured rows when a
# small image replays this graph. Leave one spare cu_seqlens slot.
metadata = self.vision_tower.vision_model.prepare_encoder_metadata(
grid_thw_list,
device=device,
max_batch_size=max_batch_size + 1,
max_seqlen_override=total_patches,
)
values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values}
values.update({k: v for k, v in metadata.items() if v is not None})
self._encoder_cg_pad_totals[values["cu_seqlens"].data_ptr()] = total_patches
return EncoderCudaGraphCaptureInputs(values=values)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
) -> "EncoderCudaGraphReplayBuffers":
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphReplayBuffers,
)
pixel_values = self._get_pixel_values(mm_kwargs)
# Unpadded: pad_cu_seqlens completes the tail with the padding sequence
# so cu_seqlens[-1] matches the captured buffer's row count.
metadata = self.vision_tower.vision_model.prepare_encoder_metadata(
self._get_grid_thw_list(mm_kwargs),
device=pixel_values.device,
)
# FlashAttention reads this CPU scalar during capture, so the captured
# launch must retain the bound for the padded buffer.
metadata.pop("max_seqlen")
values: dict[str, torch.Tensor | None] = {"pixel_values": pixel_values}
values.update(metadata)
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
inputs: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
pixel_values = inputs.pop("pixel_values")
# Remaining keys are consumed as encoder_metadata.
return self.vision_tower(pixel_values, grid_thw=None, encoder_metadata=inputs)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
return self.vision_tower(
pixel_values=self._get_pixel_values(mm_kwargs).type(
self.vision_tower.dtype
),
grid_thw=self._get_grid_thw_list(mm_kwargs),
)
+80 -13
View File
@@ -392,19 +392,33 @@ class MiniMaxVLVisionTransformer(nn.Module):
out.append([min(max_f, t - i), h, w])
return out
# ── Forward ──────────────────────────────────────────────────────────
# ── Encoder metadata (shared by eager forward and CUDA graph replay) ──
def forward(
def prepare_encoder_metadata(
self,
pixel_values: torch.Tensor,
grid_thw: list[list[int]],
) -> torch.Tensor:
# pixel_values: (total_N, C * temporal_patch_size * patch_size²)
# Output: (total_N, hidden_size)
*,
device: torch.device,
max_batch_size: int | None = None,
max_seqlen_override: int | None = None,
) -> dict[str, torch.Tensor | None]:
"""Build all grid-dependent forward inputs as standalone tensors.
hidden = self.embeddings(pixel_values) # (total_N, hidden_size)
hidden = self.pre_layrnorm(hidden)
The eager path calls this per forward; encoder CUDA graphs call it at
capture (dummy grids, padded to fixed buffer sizes) and before each
replay (real grids, unpadded — the manager's padding logic completes
the captured buffers).
Args:
grid_thw: per-item [t, h, w] patch grids (host metadata).
device: device for the attention metadata tensors.
max_batch_size: pad cu_seqlens to this many segments with empty
(zero-length) trailing segments; None leaves it unpadded.
max_seqlen_override: bake this max_seqlen instead of deriving it
from the grids; CUDA graph capture passes a worst-case value
because max_seqlen is a CPU scalar baked into the graph.
"""
limited = self._apply_max_frames_limit(grid_thw)
# Token-level cumulative sequence lengths (one segment per limited grid).
@@ -412,15 +426,23 @@ class MiniMaxVLVisionTransformer(nn.Module):
cu_seqlens_np = np.zeros(len(lens) + 1, dtype=np.int32)
np.cumsum(np.array(lens, dtype=np.int32), out=cu_seqlens_np[1:])
if max_batch_size is not None and len(lens) < max_batch_size:
pad = np.full(max_batch_size - len(lens), cu_seqlens_np[-1], dtype=np.int32)
cu_seqlens_np = np.concatenate([cu_seqlens_np, pad])
# Backend-specific encoder metadata. For FLASH_ATTN this returns the raw
# token cu_seqlens, the max segment length, and sequence_lengths=None;
# for FLASHINFER (cuDNN) it repacks cu_seqlens into element-offset
# indptrs, buckets max_seqlen, and builds padded per-sequence lengths.
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
self.attn_backend, cu_seqlens_np, hidden.device
self.attn_backend, cu_seqlens_np, device
)
max_seqlen = torch.tensor(
MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens_np),
max_seqlen_override
if max_seqlen_override is not None
else MMEncoderAttention.compute_max_seqlen(
self.attn_backend, cu_seqlens_np
),
dtype=torch.int32,
)
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
@@ -428,7 +450,7 @@ class MiniMaxVLVisionTransformer(nn.Module):
cu_seqlens_np,
self.hidden_size,
self.tp_size,
hidden.device,
device,
)
# 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for
@@ -442,6 +464,45 @@ class MiniMaxVLVisionTransformer(nn.Module):
self.spatial_merge_size,
)
return {
"rotary_cos": rotary_cos,
"rotary_sin": rotary_sin,
"cu_seqlens": cu_seqlens,
"max_seqlen": max_seqlen,
"sequence_lengths": sequence_lengths,
}
# ── Forward ──────────────────────────────────────────────────────────
def forward(
self,
pixel_values: torch.Tensor,
grid_thw: list[list[int]] | None = None,
*,
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
) -> torch.Tensor:
# pixel_values: (total_N, C * temporal_patch_size * patch_size²)
# Output: (total_N, hidden_size)
hidden = self.embeddings(pixel_values) # (total_N, hidden_size)
hidden = self.pre_layrnorm(hidden)
if encoder_metadata is None:
assert grid_thw is not None
encoder_metadata = self.prepare_encoder_metadata(
grid_thw, device=hidden.device
)
cu_seqlens = encoder_metadata["cu_seqlens"]
rotary_cos = encoder_metadata["rotary_cos"]
rotary_sin = encoder_metadata["rotary_sin"]
max_seqlen = encoder_metadata["max_seqlen"]
sequence_lengths = encoder_metadata.get("sequence_lengths")
assert cu_seqlens is not None
assert rotary_cos is not None
assert rotary_sin is not None
assert max_seqlen is not None
# Encoder expects (N, 1, hidden_size) — add batch dim
hidden = hidden.unsqueeze(1)
@@ -600,9 +661,15 @@ class MiniMaxVLVisionModel(nn.Module):
def forward(
self,
pixel_values: torch.Tensor,
grid_thw: list[list[int]],
grid_thw: list[list[int]] | None = None,
*,
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
) -> torch.Tensor:
hidden = self.vision_model(pixel_values=pixel_values, grid_thw=grid_thw)
hidden = self.vision_model(
pixel_values=pixel_values,
grid_thw=grid_thw,
encoder_metadata=encoder_metadata,
)
if hidden.dim() == 3:
hidden = hidden.squeeze(0)
hidden = self.multi_modal_projector(hidden)
+8 -1
View File
@@ -70,6 +70,9 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model
from vllm.models.minimax_m3.common.encoder_cudagraph import (
MiniMaxM3EncoderCudaGraphMixin,
)
from vllm.models.minimax_m3.common.indexer import MiniMaxM3Indexer
from vllm.models.minimax_m3.common.mm_preprocess import (
MiniMaxM3VLDummyInputsBuilder,
@@ -1076,7 +1079,11 @@ class MiniMaxM3SparseForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
dummy_inputs=MiniMaxM3VLDummyInputsBuilder,
)
class MiniMaxM3SparseForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3
nn.Module,
SupportsMultiModal,
MiniMaxM3EncoderCudaGraphMixin,
SupportsPP,
SupportsEagle3,
):
"""Top-level (VL) entry point for MiniMax M3.