[MM][CG] Support ViT full CUDA graph for Ernie-4.5-VL image inference (#45254)

Signed-off-by: Qiuyang Yue <yueqiuyang1389@gmail.com>
This commit is contained in:
Qiuyang Yue
2026-08-06 05:04:19 +00:00
committed by GitHub
parent 821717118f
commit 777b01d1a8
4 changed files with 288 additions and 12 deletions
+1
View File
@@ -118,6 +118,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra
| Architecture | Models | CG for Image | CG for Video | Dual-Path Graph |
| ------------ | ------ | ------------ | ------------ | --------------- |
| `DeepseekOCRForCausalLM` | `DeepSeek-OCR` | ✅︎ | ❌︎ | ✅︎ |
| `Ernie4_5_VLMoeForConditionalGeneration` | `ERNIE-4.5-VL` | ✅︎ | ❌︎ | ❌︎ |
| `Gemma3ForConditionalGeneration` | `Gemma3` | ✅︎ | ❌︎ | ❌︎ |
| `Glm4vForConditionalGeneration` | `GLM-4.1V, GLM-4.6V-Flash` | ✅︎ | ✅︎ | ❌︎ |
| `Gemma4ForConditionalGeneration` | `Gemma-4` | ✅︎ | ✅︎ | ❌︎ |
@@ -2426,6 +2426,7 @@ MODELS_SUPPORT_VIT_CUDA_GRAPH = [
"stepvl",
"glm4_1v",
"deepseek_ocr",
"ernie45_vl",
]
@@ -66,6 +66,14 @@ def gemma3_chat_template(content: str) -> str:
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
def ernie45_vl_chat_template(content: str) -> str:
return (
f"<|begin_of_sentence|>User: {content}"
"Picture 1:<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>\n"
"Assistant: <think></think>"
)
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
"gemma3": VitCudagraphTestConfig(
model="google/gemma-3-4b-it",
@@ -208,6 +216,25 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
),
},
),
"ernie45_vl": VitCudagraphTestConfig(
model="baidu/ERNIE-4.5-VL-28B-A3B-PT",
# Image only: Ernie's resampler applies a temporal conv for video
# that changes the output token count, so video uses the eager path.
modalities=["image"],
image_prompt=ernie45_vl_chat_template("What is in this image?"),
# Shrink to 1 text + 1 vision layer with random weights so the test
# runs on any CI GPU and skips the ~56 GiB weight download. The test
# only validates encoder CG capture/replay, not output quality.
vllm_runner_kwargs={
"load_format": "dummy",
"trust_remote_code": True,
"revision": "refs/pr/17",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="Ernie4_5_VLMoeForConditionalGeneration",
),
},
),
"glm4_1v": VitCudagraphTestConfig(
model="zai-org/GLM-4.1V-9B-Thinking",
image_prompt=(
+259 -12
View File
@@ -77,12 +77,18 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .ernie45_vl_moe import Ernie4_5_VLMoeForCausalLM
from .interfaces import (
MultiModalEmbeddings,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsPP,
)
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
from .utils import (
AutoWeightsLoader,
WeightsMapper,
maybe_prefix,
scatter_output_slices,
)
from .vision import get_vit_attn_backend
logger = init_logger(__name__)
@@ -453,13 +459,43 @@ class Ernie4_5_VisionTransformer(nn.Module):
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
return max_seqlen
def forward(
self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, num_pad=0
) -> torch.Tensor:
hidden_states = self.patch_embed(hidden_states)
def prepare_encoder_metadata(
self,
grid_thw: torch.Tensor | list[list[int]],
*,
num_pad: int = 0,
max_batch_size: int | None = None,
max_seqlen_override: int | None = None,
device: torch.device | None = None,
) -> dict[str, torch.Tensor]:
"""Compute encoder metadata outside the CUDA graph.
rotary_pos_emb = self.rot_pos_emb(grid_thw)
rotary_pos_emb = rotary_pos_emb.to(hidden_states.device)
Splits the rotary embeddings, ``cu_seqlens`` and ``max_seqlen`` out of
`forward` so they can be precomputed on the host side and fed into a
captured graph through fixed buffers. Shared by the eager path, CUDA
graph capture and CUDA graph replay
Args:
grid_thw: Per-frame grid sizes as a ``[num_frames, 3]`` tensor or
an equivalent list of ``[t, h, w]``.
num_pad: Legacy ``cu_seqlens`` padding
max_batch_size: If set, pad ``cu_seqlens`` to ``max_batch_size + 1``
entries for a fixed graph buffer shape.
max_seqlen_override: If set, use this value for ``max_seqlen``
instead of deriving it from ``cu_seqlens``. CUDA graph capture
bakes ``max_seqlen`` at capture time, so it must cover the
worst-case replay.
device: Device for the returned tensors. Defaults to ``self.device``.
Returns:
Dict with ``rotary_pos_emb``, ``cu_seqlens`` and ``max_seqlen``.
"""
if device is None:
device = self.device
if not isinstance(grid_thw, torch.Tensor):
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
rotary_pos_emb = self.rot_pos_emb(grid_thw).to(device)
cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
@@ -472,14 +508,60 @@ class Ernie4_5_VisionTransformer(nn.Module):
else:
cu_seqlens = torch.cat([zeros, cu_seqlens])
# Pad cu_seqlens to a fixed number of sequences so the CUDA graph
# replay buffer keeps a constant shape across batches.
if max_batch_size is not None:
num_seqs = cu_seqlens.numel() - 1
if num_seqs < max_batch_size:
cu_seqlens = torch.cat(
[
cu_seqlens,
cu_seqlens.new_full(
(max_batch_size - num_seqs,), cu_seqlens[-1]
),
]
)
# max_seqlen is baked into the graph at capture time, so capture passes
# a worst-case override. compute_attn_mask_seqlen keeps it on CPU
# (consumed via .item()), avoiding a captured D2H copy.
if max_seqlen_override is not None:
max_seqlen = torch.tensor(max_seqlen_override, dtype=torch.int32)
else:
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
cu_seqlens = cu_seqlens.to(device)
return {
"rotary_pos_emb": rotary_pos_emb,
"cu_seqlens": cu_seqlens,
"max_seqlen": max_seqlen,
}
def forward(
self,
hidden_states: torch.Tensor,
grid_thw: torch.Tensor | None = None,
num_pad: int = 0,
*,
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
hidden_states = self.patch_embed(hidden_states)
if encoder_metadata is None:
# Eager path: compute metadata inline
encoder_metadata = self.prepare_encoder_metadata(
grid_thw, num_pad=num_pad, device=hidden_states.device
)
rotary_pos_emb = encoder_metadata["rotary_pos_emb"]
cu_seqlens = encoder_metadata["cu_seqlens"]
max_seqlen = encoder_metadata["max_seqlen"]
# add batch size
if hidden_states.ndim == 2:
hidden_states = hidden_states.unsqueeze(dim=1)
# pre-compute max_seqlen for attn mask to reduce cuMemcpy operations
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
for i, blk in enumerate(self.blocks):
for blk in self.blocks:
hidden_states = blk(
hidden_states,
cu_seqlens=cu_seqlens,
@@ -1256,7 +1338,12 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
dummy_inputs=Ernie4_5_VLDummyInputsBuilder,
)
class Ernie4_5_VLMoeForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
nn.Module,
SupportsMultiModal,
SupportsLoRA,
SupportsPP,
SupportsMRoPE,
SupportsEncoderCudaGraph,
):
packed_modules_mapping = {
"qkv_proj": [
@@ -1489,6 +1576,166 @@ class Ernie4_5_VLMoeForConditionalGeneration(
video_grid_thw=video_grid_thw,
)
# -- SupportsEncoderCudaGraph protocol methods --
#
# Only the vision transformer (self.vision_model) is captured into the
# CUDA graph. The VariableResolutionResamplerModel (spatial merge +
# projection) runs eagerly in encoder_eager_forward / postprocess_
# encoder_output because its temporal path performs host-side (numpy)
# indexing that cannot be captured. Image only: video uses a temporal
# conv that changes the output token count, so it falls back to the
# eager multimodal path for now.
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
return EncoderCudaGraphConfig(
modalities=["image"],
# Ernie consumes a single rotary freqs tensor (not cos/sin)
buffer_keys=[
"pixel_values",
"rotary_pos_emb",
"cu_seqlens",
"max_seqlen",
],
# Post-merge embeddings, produced in postprocess_encoder_output,
# are at the LM hidden dim, used only for DP gather sizing.
out_hidden_size=self.config.hidden_size,
)
def get_encoder_cudagraph_budget_range(self, vllm_config) -> tuple[int, int]:
# Min: a 224x224 image -> 16x16 patches (patch_size=14),
# spatial_merge_size=2 -> 8x8 = 64 output tokens.
min_budget = 64
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
vllm_config.model_config.max_model_len,
)
return (min_budget, max_budget)
def get_encoder_cudagraph_item_specs(self, mm_kwargs: dict[str, Any]):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
m = self.vision_model.spatial_merge_size
grid_thw = mm_kwargs["image_grid_thw"].tolist()
return [
EncoderItemSpec(
input_size=t * h * w,
output_tokens=(t * h * w) // m // m,
)
for t, h, w in grid_thw
]
def select_encoder_cudagraph_items(
self, mm_kwargs: dict[str, Any], indices: list[int]
) -> dict[str, Any]:
grid_thw = mm_kwargs["image_grid_thw"]
pixel_values = mm_kwargs["pixel_values"]
if len(indices) == 0:
return {"pixel_values": pixel_values[:0], "image_grid_thw": grid_thw[:0]}
# Cumulative patch offsets for slicing the concatenated pixel_values.
patches_per_item = (grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).tolist()
cum_patches = [0]
for p in patches_per_item:
cum_patches.append(cum_patches[-1] + p)
selected_pv = torch.cat(
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
)
return {"pixel_values": selected_pv, "image_grid_thw": grid_thw[indices]}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget,
max_batch_size,
max_frames_per_batch,
device,
dtype,
path: str = "default",
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
)
m = self.vision_model.spatial_merge_size
# Ceil so the buffer fits the worst case of one item using the full
# budget.
per_mm_item_output = (token_budget + max_batch_size - 1) // max_batch_size
# Image-format grid (T=1): h=m, w=per_mm_item_output*m, so the merged
# token count per item equals per_mm_item_output.
grid_config = [[1, m, per_mm_item_output * m] for _ in range(max_batch_size)]
patch_embed = self.vision_model.patch_embed
in_channels = patch_embed.in_channels
patch_size = patch_embed.patch_size
total_patches = sum(t * h * w for t, h, w in grid_config)
flattened_patch_size = in_channels * patch_size * patch_size
dummy_pixel_values = torch.randn(
total_patches, flattened_patch_size, device=device, dtype=dtype
)
# max_seqlen is baked at capture: worst case is one item consuming the
# full budget - seq_len = token_budget * spatial_merge_size**2.
metadata = self.vision_model.prepare_encoder_metadata(
grid_config,
max_batch_size=max_batch_size,
max_seqlen_override=token_budget * (m**2),
device=device,
)
values = metadata | {"pixel_values": dummy_pixel_values}
return EncoderCudaGraphCaptureInputs(values=values)
def prepare_encoder_cudagraph_replay_buffers(
self, mm_kwargs, max_batch_size, max_frames_per_batch, path: str = "default"
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphReplayBuffers,
)
metadata = self.vision_model.prepare_encoder_metadata(
mm_kwargs["image_grid_thw"].tolist(), max_batch_size=max_batch_size
)
values = metadata | {
"pixel_values": mm_kwargs["pixel_values"],
}
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self, values: dict[str, torch.Tensor], path: str = "default"
) -> torch.Tensor:
# Graph captures the ViT only, and the resampler runs in
# postprocess_encoder_output.
pixel_values = values.pop("pixel_values")
return self.vision_model(pixel_values, encoder_metadata=values)
def encoder_eager_forward(
self, mm_kwargs: dict[str, Any], path: str = "default"
) -> torch.Tensor:
# Eager fallback: run the full pipeline (ViT + resampler). The result
# is scattered directly, so it must be the post-merge embeddings.
pixel_values = mm_kwargs["pixel_values"].type(self.vision_model.dtype)
grid_thw = mm_kwargs["image_grid_thw"].to(self.vision_model.device)
image_features = self.vision_model(pixel_values, grid_thw)
return self.resampler_model(image_features, grid_thw)
def postprocess_encoder_output(
self,
output: torch.Tensor,
indices: list[int],
per_item_out_tokens: list[int],
dest,
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
) -> None:
# The graph output is the raw ViT features (pre-merge), padded to the
# token budget. Run the resampler eagerly on the valid portion using
# the actual batch grid_thw, then scatter the post-merge embeddings.
grid_thw = batch_mm_kwargs["image_grid_thw"].to(output.device)
num_valid = int((grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).sum())
image_embeds = self.resampler_model(output[:num_valid], grid_thw)
scatter_output_slices(image_embeds, indices, per_item_out_tokens, dest, clone)
def _process_image_input(
self, image_input: Ernie4_5_VLImageInputs
) -> tuple[torch.Tensor, ...]: