[MM][Perf][CG] Support ViT full CUDA graph for glm4_1v image and video inference (#40576)

Signed-off-by: grYe99 <guorongye99@gmail.com>
Co-authored-by: grYe99 <guorongye99@gmail.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
GuorongYe
2026-06-09 11:13:56 +08:00
committed by GitHub
co-authored by grYe99 Isotr0py
parent d3de61502f
commit 9f153aa781
4 changed files with 480 additions and 25 deletions
+1
View File
@@ -88,6 +88,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra
| `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ |
| `Qwen3_5ForConditionalGeneration` | `Qwen3.5` | ✅︎ | ✅︎ |
| `Step3VLForConditionalGeneration` | `Step3-VL` | ✅︎ | ❌︎ |
| `Glm4vForConditionalGeneration` | `GLM-4.1V, GLM-4.6V-Flash` | ✅︎ | ✅︎ |
!!! note
Encoder CUDA Graphs have currently been tested with `--mm-encoder-attn-backend=FLASH_ATTN` and `--mm-encoder-attn-backend=FLASHINFER` on Blackwell GPUs.
@@ -2562,6 +2562,7 @@ MODELS_SUPPORT_VIT_CUDA_GRAPH = [
"qwen3_5",
"qwen3_5_moe",
"stepvl",
"glm4_1v",
]
@@ -137,6 +137,28 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
),
},
),
"glm4_1v": VitCudagraphTestConfig(
model="zai-org/GLM-4.1V-9B-Thinking",
image_prompt=(
"[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
"<|begin_of_image|><|image|><|end_of_image|>"
"What is in this image?<|assistant|>assistant\n"
),
video_prompt=(
"[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
"<|begin_of_video|><|video|><|end_of_video|>"
"Describe this video in one sentence<|assistant|>assistant\n"
),
needs_video_metadata=True,
marks=[pytest.mark.core_model],
vllm_runner_kwargs={
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="Glm4vForConditionalGeneration",
),
},
),
}
+456 -25
View File
@@ -97,10 +97,12 @@ from vllm.multimodal.processing import (
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
from ..layers.activation import SiluAndMul
from .interfaces import (
MultiModalEmbeddings,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
@@ -626,6 +628,11 @@ class Glm4vVisionTransformer(nn.Module):
) -> None:
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.tp_size = (
1 if use_data_parallel else get_tensor_model_parallel_world_size()
)
patch_size = vision_config.patch_size
temporal_patch_size = vision_config.temporal_patch_size
in_channels = vision_config.in_channels
@@ -701,7 +708,7 @@ class Glm4vVisionTransformer(nn.Module):
return self.patch_embed.proj.weight.device
def rot_pos_emb(
self, grid_thw: torch.Tensor
self, grid_thw: list[list[int]]
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
pos_ids = []
for t, h, w in grid_thw:
@@ -729,7 +736,7 @@ class Glm4vVisionTransformer(nn.Module):
)
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
pos_ids = torch.cat(pos_ids, dim=0)
max_grid_size = grid_thw[:, 1:].max()
max_grid_size = max(max(h, w) for _, h, w in grid_thw)
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
@@ -752,45 +759,192 @@ class Glm4vVisionTransformer(nn.Module):
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
return max_seqlen
def pos_embeds_interpolate(self, grid_thw: list[list[int]]) -> torch.Tensor:
"""Pre-compute absolute position embeddings for all input samples.
The original `self.embeddings` fused token embeddings and position embeddings
in one call, which prevented preparing position embeddings as static metadata
required by CUDA graph capture / replay. This method decouples the two by
feeding an all-zero token tensor to `self.embeddings`. The module therefore only
performs bicubic interpolation based on the coordinates and returns pure
position embeddings. These are cached in `prepare_encoder_metadata` and later
added to the patch tokens in `forward` via `x = x + pos_embeds`, keeping the
forward graph compatible with CUDA graph replay. Coordinate generation matches
`rot_pos_emb` exactly to guarantee spatial alignment.
"""
device = self.embeddings.position_embedding.weight.device
dtype = self.dtype
all_embeds = []
for t, h, w in grid_thw:
# Use the same coordinate generation logic as rot_pos_emb
# to ensure consistent positional embedding interpolation
h_coords = torch.arange(h).unsqueeze(1).expand(-1, w)
w_coords = torch.arange(w).unsqueeze(0).expand(h, -1)
h_coords = (
h_coords.reshape(
h // self.spatial_merge_size,
self.spatial_merge_size,
w // self.spatial_merge_size,
self.spatial_merge_size,
)
.permute(0, 2, 1, 3)
.flatten()
)
w_coords = (
w_coords.reshape(
h // self.spatial_merge_size,
self.spatial_merge_size,
w // self.spatial_merge_size,
self.spatial_merge_size,
)
.permute(0, 2, 1, 3)
.flatten()
)
lengths = [h * w] * t
image_shapes = torch.tensor([[t, h, w]], device=device)
h_coords_repeated = h_coords.repeat(t)
w_coords_repeated = w_coords.repeat(t)
embeds = self.embeddings(
embeddings=torch.zeros(
h * w * t, self.hidden_size, device=device, dtype=dtype
),
lengths=lengths,
image_shapes=image_shapes,
h_coords=h_coords_repeated,
w_coords=w_coords_repeated,
)
all_embeds.append(embeds)
return torch.cat(all_embeds, dim=0).to(dtype)
def prepare_encoder_metadata(
self,
grid_thw_list: list[list[int]],
*,
max_batch_size: int | None = None,
max_frames_per_batch: int | None = None,
max_seqlen_override: int | None = None,
device: torch.device | None = None,
) -> dict[str, torch.Tensor | None]:
"""Compute encoder metadata from grid_thw_list.
Shared by the eager forward path, CUDA graph capture, and
CUDA graph replay to avoid duplicated implementation.
Args:
grid_thw_list: Grid configurations as list of [t, h, w].
max_batch_size: If set, pad cu_seqlens to this size
(needed for CUDA graph capture/replay).
max_frames_per_batch: If set, overrides max_batch_size for
cu_seqlens padding. For video inputs each item contributes
T attention sequences (frames); this sizes the buffer to
the total frame budget so video replays never overflow.
max_seqlen_override: If set, use this value for max_seqlen
instead of computing from cu_seqlens (needed for CUDA
graph capture to cover worst-case replay scenarios).
device: Device to place tensors on. Defaults to self.device.
"""
if device is None:
device = self.device
metadata: dict[str, torch.Tensor | None] = {}
# Positional embeddings
metadata["pos_embeds"] = self.pos_embeds_interpolate(grid_thw_list)
rotary_cos, rotary_sin, _ = self.rot_pos_emb(grid_thw_list)
metadata["rotary_pos_emb_cos"] = rotary_cos
metadata["rotary_pos_emb_sin"] = rotary_sin
# cu_seqlens from grid_thw
grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
dtype=np.int32
)
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
# Pad cu_seqlens to the required number of sequences.
# For videos each item contributes T frames = T attention sequences,
# so the total can exceed max_batch_size. max_frames_per_batch
# overrides the pad target when set.
pad_to = (
max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
)
if pad_to is not None:
num_seqs = len(cu_seqlens) - 1
if num_seqs < pad_to:
cu_seqlens = np.concatenate(
[
cu_seqlens,
np.full(
pad_to - num_seqs,
cu_seqlens[-1],
dtype=np.int32,
),
]
)
# sequence_lengths (backend-specific)
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
self.attn_backend, cu_seqlens, device
)
# max_seqlen
if max_seqlen_override is not None:
max_seqlen_val = max_seqlen_override
else:
max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
self.attn_backend, cu_seqlens
)
# Keep max_seqlen on CPU: attention wrappers call .item() on it,
# and having it on GPU would capture a wasteful D2H copy in CUDA
# graphs without changing behavior (the scalar is baked at capture).
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
# Recompute cu_seqlens (backend-specific transformation)
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
self.attn_backend,
cu_seqlens,
self.hidden_size,
self.tp_size,
device,
)
return metadata
def forward(
self,
x: torch.Tensor,
grid_thw: torch.Tensor | list[list[int]],
*,
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
if isinstance(grid_thw, list):
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
if encoder_metadata is None:
if not isinstance(grid_thw, list):
grid_thw = grid_thw.tolist()
encoder_metadata = self.prepare_encoder_metadata(grid_thw)
# patchify
x = x.to(device=self.device, dtype=self.dtype)
x = self.patch_embed(x)
x = self.post_conv_layernorm(x)
# compute position embedding
rotary_pos_emb_cos, rotary_pos_emb_sin, image_type_ids = self.rot_pos_emb(
grid_thw
)
# compute cu_seqlens
cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
).cumsum(dim=0, dtype=torch.int32)
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
# pre-compute max_seqlen for attn mask to reduce cuMemcpy operations
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
x = self.embeddings(
x, seqlens, grid_thw, image_type_ids[:, 0], image_type_ids[:, 1]
)
pos_embeds = encoder_metadata["pos_embeds"]
x = x + pos_embeds
# transformers
x = x.unsqueeze(1)
for blk in self.blocks:
x = blk(
x,
cu_seqlens=cu_seqlens,
rotary_pos_emb_cos=rotary_pos_emb_cos,
rotary_pos_emb_sin=rotary_pos_emb_sin,
max_seqlen=max_seqlen,
cu_seqlens=encoder_metadata["cu_seqlens"],
rotary_pos_emb_cos=encoder_metadata["rotary_pos_emb_cos"],
rotary_pos_emb_sin=encoder_metadata["rotary_pos_emb_sin"],
max_seqlen=encoder_metadata["max_seqlen"],
)
# adapter
@@ -1587,7 +1741,12 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
dummy_inputs=Glm4vDummyInputsBuilder,
)
class Glm4vForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
nn.Module,
SupportsMultiModal,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsPP,
SupportsMRoPE,
):
packed_modules_mapping = {
"qkv_proj": [
@@ -1625,8 +1784,12 @@ class Glm4vForConditionalGeneration(
multimodal_config = vllm_config.model_config.multimodal_config
self.config = config
self.model_config = vllm_config.model_config
self.multimodal_config = multimodal_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.is_multimodal_pruning_enabled = (
multimodal_config.is_multimodal_pruning_enabled()
)
with self._mark_tower_model(vllm_config, {"image", "video"}):
self.visual = Glm4vVisionTransformer(
@@ -1752,6 +1915,274 @@ class Glm4vForConditionalGeneration(
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
return video_embeds.split(sizes)
# -- SupportsEncoderCudaGraph protocol methods --
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphConfig,
)
# When EVS pruning is enabled, embed_multimodal post-processes both
# image and video embeddings (mrope positions are appended for image,
# prune+append for video). The encoder CUDA graph path bypasses that
# post-process, producing inconsistent embedding formats vs eager. So
# disable CUDA graph for all modalities when pruning is on.
modalities = [] if self.is_multimodal_pruning_enabled else ["image", "video"]
# Compute max_frames_per_video for budget sizing.
max_frames = self.get_max_frames_per_video() if "video" in modalities else 1
return EncoderCudaGraphConfig(
modalities=modalities,
buffer_keys=[
"pixel_values",
"pos_embeds",
"rotary_pos_emb_cos",
"rotary_pos_emb_sin",
"cu_seqlens",
"max_seqlen",
"sequence_lengths",
],
out_hidden_size=self.visual.out_hidden_size,
max_frames_per_video=max_frames,
)
def get_input_modality(
self,
mm_kwargs: dict[str, Any],
) -> str:
if "image_grid_thw" in mm_kwargs:
return "image"
elif "video_grid_thw" in mm_kwargs:
return "video"
raise AssertionError("This line should be unreachable.")
def get_max_frames_per_video(self) -> int:
mm_registry = MULTIMODAL_REGISTRY
info = mm_registry.get_processing_info(self.model_config)
max_frames_per_video = info.get_num_frames_with_most_features(
seq_len=self.model_config.max_model_len,
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
)
# Small 'max_frames_per_video' will cause 'tensor mismatch' in PR#43403
# 16 is the default 'num_frames' of '_get_vision_info'
return max(max_frames_per_video, 16)
def get_encoder_cudagraph_budget_range(
self,
vllm_config,
) -> tuple[int, int]:
# Min: estimated smallest possible encoder input.
# 224x224 image → 16x16 patches (patch_size=14)
# spatial_merge_size=2 → 8x8 = 64 tokens
min_budget = 64
# Max: capped by max_num_batched_tokens
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
vllm_config.model_config.max_model_len,
)
return (min_budget, max_budget)
def _get_pixel_values_by_modality(
self,
mm_kwargs: dict[str, Any],
) -> torch.Tensor:
if self.get_input_modality(mm_kwargs) == "image":
pixel_values = mm_kwargs["pixel_values"]
else:
pixel_values = mm_kwargs["pixel_values_videos"]
return pixel_values
def _get_grid_thw_by_modality(
self,
mm_kwargs: dict[str, Any],
) -> list[tuple[int, int, int]]:
grid_thw_key = f"{self.get_input_modality(mm_kwargs)}_grid_thw"
grid_thw = mm_kwargs[grid_thw_key]
if not isinstance(grid_thw, list):
grid_thw = grid_thw.tolist()
return grid_thw
def get_encoder_cudagraph_item_specs(
self,
mm_kwargs: dict[str, Any],
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
m = self.visual.spatial_merge_size
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
return [
EncoderItemSpec(
input_size=t * h * w,
output_tokens=t * (h // m) * (w // m),
)
for t, h, w in grid_thw
]
def select_encoder_cudagraph_items(
self,
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
if len(indices) == 0:
if self.get_input_modality(mm_kwargs) == "image":
return {
"pixel_values": pixel_values[:0],
"image_grid_thw": [],
}
else:
return {
"pixel_values_videos": pixel_values[:0],
"video_grid_thw": [],
}
# Compute cumulative patch offsets for slicing pixel_values
patches_per_item = [t * h * w for t, h, w in grid_thw]
cum_patches = [0]
for p in patches_per_item:
cum_patches.append(cum_patches[-1] + p)
selected_pv = torch.cat(
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
)
selected_grid = [grid_thw[i] for i in indices]
if self.get_input_modality(mm_kwargs) == "image":
return {
"pixel_values": selected_pv,
"image_grid_thw": selected_grid,
}
else:
return {
"pixel_values_videos": selected_pv,
"video_grid_thw": selected_grid,
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
)
spatial_merge_size = self.visual.spatial_merge_size
per_mm_item_output = token_budget // max_batch_size
frames_per_item = max_frames_per_batch // max_batch_size
if frames_per_item > 1:
# Build the capture grid using a video-format layout so that
# cu_seqlens is sized for video replays from the start.
# cu_seqlens has one entry per attention sequence (one per frame),
# so using T > 1 per item makes the buffer large enough without
# relying solely on padding.
# Ceiling ensures frames_per_item * tokens_per_frame >= per_mm_item_output
# so the pixel_values buffer covers any valid single-item replay.
tokens_per_frame = (
per_mm_item_output + frames_per_item - 1
) // frames_per_item
# Video-format grid_config (T=frames_per_item).
grid_config = [
[
frames_per_item,
spatial_merge_size,
tokens_per_frame * spatial_merge_size,
]
for _ in range(max_batch_size)
]
else:
# Image-format grid_config (T=1).
grid_config = [
[1, spatial_merge_size, per_mm_item_output * spatial_merge_size]
for _ in range(max_batch_size)
]
# Create dummy pixel_values
patch_embed = self.visual.patch_embed
in_channels = patch_embed.proj.in_channels
patch_size = patch_embed.patch_size
temporal_patch_size = patch_embed.temporal_patch_size
total_patches = sum(t * h * w for t, h, w in grid_config)
flattened_patch_size = (
in_channels * temporal_patch_size * patch_size * patch_size
)
dummy_pixel_values = torch.randn(
total_patches, flattened_patch_size, device=device, dtype=dtype
)
# Override max_seqlen with a safe upper bound for capture.
# max_seqlen.item() gets baked into the CUDA graph (not replayed),
# so the capture value must cover any replay scenario.
# Worst case: 1 item consuming the full budget ->
# seq_len = token_budget * spatial_merge_size^2.
metadata = self.visual.prepare_encoder_metadata(
grid_config,
max_batch_size=max_batch_size,
max_frames_per_batch=max_frames_per_batch,
max_seqlen_override=token_budget * (spatial_merge_size**2),
device=device,
)
# Just use image-modality dummy input_buffer for capturing, since it's also
# compatible for video inputs (has the same shape: [num_patches, C*T*P*P]).
values = metadata | {
"pixel_values": dummy_pixel_values,
}
return EncoderCudaGraphCaptureInputs(
values=values,
)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
):
modality = self.get_input_modality(mm_kwargs)
grid_thw_list = self._get_grid_thw_by_modality(mm_kwargs)
if modality == "image":
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_batch_size=max_batch_size,
)
elif modality == "video":
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_frames_per_batch=max_frames_per_batch,
)
else:
raise AssertionError("This line should be unreachable.")
values = metadata | {
"pixel_values": self._get_pixel_values_by_modality(mm_kwargs),
}
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
values: dict[str, torch.Tensor],
) -> torch.Tensor:
pixel_values = values.pop("pixel_values")
metadata = values
return self.visual(pixel_values, None, encoder_metadata=metadata)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
) -> torch.Tensor:
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
return self.visual(pixel_values, grid_thw)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}