[MM][Perf][CG] Support ViT full CUDA graph for Kimi-VL (#41992)

Signed-off-by: oguz <oguzhankir17@gmail.com>
This commit is contained in:
Oğuzhan KIR
2026-06-17 12:14:01 +00:00
committed by GitHub
parent e28e8c8782
commit fa85ead2f3
5 changed files with 504 additions and 45 deletions
+272 -43
View File
@@ -45,7 +45,9 @@
from collections.abc import Sequence
from copy import deepcopy
from functools import cached_property
from typing import Any
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
@@ -110,23 +112,42 @@ class Learnable2DInterpPosEmb(nn.Module):
def reset_parameters(self):
nn.init.normal_(self.weight)
def forward(self, x: torch.Tensor, grid_hws: torch.Tensor) -> torch.Tensor:
pos_embs = []
for shape in grid_hws.tolist():
if shape == self.weight.shape[:-1]:
def get_pos_embeds(
self,
grid_hws_list: list[list[int]] | list[tuple[int, int]],
) -> torch.Tensor:
"""Build packed per-token positional embeddings for a list of grids.
Returns a tensor of shape ``(sum(h * w), dim)`` formed by interpolating
the learned ``(height, width, dim)`` weight to each ``(h, w)`` grid and
concatenating the flattened results in the same order as
``grid_hws_list``. Lives outside the captured CUDA graph so the
per-grid Python iteration is safe.
"""
weight_shape = list(self.weight.shape[:-1])
pos_embs: list[torch.Tensor] = []
for shape in grid_hws_list:
shape_list = [int(shape[0]), int(shape[1])]
if shape_list == weight_shape:
pos_embs.append(self.weight.flatten(end_dim=1))
else:
pos_embs.append(
F.interpolate(
self.weight.permute((2, 0, 1)).unsqueeze(0),
size=shape,
size=tuple(shape_list),
mode=self.interpolation_mode,
)
.squeeze(0)
.permute((1, 2, 0))
.flatten(end_dim=1)
)
out = x + torch.cat(pos_embs)
if not pos_embs:
return self.weight.new_zeros((0, self.weight.shape[-1]))
return torch.cat(pos_embs)
def forward(self, x: torch.Tensor, grid_hws: torch.Tensor) -> torch.Tensor:
pos_embs = self.get_pos_embeds(grid_hws.tolist())
out = x + pos_embs
return out
@@ -158,19 +179,29 @@ class MoonVisionPatchEmbed(nn.Module):
height=pos_emb_height, width=pos_emb_width, dim=out_dim
)
def forward(self, x: torch.Tensor, grid_hw: torch.Tensor) -> torch.Tensor:
def forward(
self,
x: torch.Tensor,
grid_hw: torch.Tensor | None = None,
*,
pos_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Args:
x (L, Channels): input tensor
grid_hw (N, 2): grid height and width
pos_embeds: precomputed positional embeddings of shape
``(L, Cout)``. When provided, ``grid_hw`` is unused and the
CUDA-graph-incompatible interpolation in ``self.pos_emb`` is
skipped.
Returns:
(L, Cout) tensor
"""
x = self.proj(x).view(x.size(0), -1)
# apply positional embedding
x = self.pos_emb(x, grid_hw)
return x
if pos_embeds is not None:
return x + pos_embeds
return self.pos_emb(x, grid_hw)
class Rope2DPosEmb(nn.Module):
@@ -243,6 +274,35 @@ class Rope2DPosEmb(nn.Module):
freqs_cis = freqs_cis.reshape(self.max_height, self.max_width, -1)
return freqs_cis
def get_freqs_cis_by_seqlens_list(
self,
grid_hws_list: list[list[int]] | list[tuple[int, int]],
) -> torch.Tensor:
"""List-based variant of :meth:`get_freqs_cis_by_seqlens`.
Accepts a Python list of ``(h, w)`` pairs so callers that already
operate outside the captured CUDA graph can avoid materializing a
tensor + ``.tolist()`` round-trip.
"""
assert all(
1 <= h <= self.max_height and 1 <= w <= self.max_width
for h, w in grid_hws_list
), (
grid_hws_list,
self.max_height,
self.max_width,
)
if not grid_hws_list:
return self.precomputed_freqs_cis.new_zeros((0, self.dim // 2))
freqs_cis = torch.cat(
[
self.precomputed_freqs_cis[:h, :w].reshape(-1, self.dim // 2)
for h, w in grid_hws_list
],
dim=0,
)
return freqs_cis
def get_freqs_cis_by_seqlens(self, grid_hws: torch.Tensor) -> torch.Tensor:
"""
Args:
@@ -250,22 +310,7 @@ class Rope2DPosEmb(nn.Module):
Returns:
freqs_cis: tensor of shape (sum(t * height * width), dim//2)
"""
shapes = grid_hws.tolist()
assert all(
1 <= h <= self.max_height and 1 <= w <= self.max_width for h, w in shapes
), (
shapes,
self.max_height,
self.max_width,
)
freqs_cis = torch.cat(
[
self.precomputed_freqs_cis[:h, :w].reshape(-1, self.dim // 2)
for h, w in shapes
],
dim=0,
)
return freqs_cis
return self.get_freqs_cis_by_seqlens_list(grid_hws.tolist())
def get_freqs_cis_by_idx(
self, pos_idx: torch.Tensor, pos_idx_mask: torch.Tensor
@@ -392,11 +437,15 @@ class MoonVitEncoderLayer(nn.Module):
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rope_freqs_cis: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
):
"""
Args:
x (torch.Tensor): (seqlen, hidden_dim)
cu_seqlens (torch.Tensor):
max_seqlen: Optional precomputed scalar tensor. When omitted it
is derived from ``cu_seqlens``, which produces a GPU scalar
that breaks CUDA graph capture.
"""
seq_length = x.size(0)
xqkv, _ = self.wqkv(x)
@@ -412,7 +461,8 @@ class MoonVitEncoderLayer(nn.Module):
xq, xk = apply_rope(xq, xk, rope_freqs_cis)
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
if max_seqlen is None:
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
attn_out = self.attn(
xq.unsqueeze(0),
xk.unsqueeze(0),
@@ -433,10 +483,12 @@ class MoonVitEncoderLayer(nn.Module):
hidden_states: torch.Tensor,
cu_seqlens: torch.Tensor,
rope_freqs_cis: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Args:
hidden_states: non-packed (B, N, D) or packed (L, D). if non-packed, seqlens should be None, if packed, seqlens should be set
max_seqlen: optional precomputed max-sequence-length scalar.
Returns:
output: same shape of input, non-packed (B, N, D) for non-packed input, (L, D) for packed input
@@ -444,7 +496,10 @@ class MoonVitEncoderLayer(nn.Module):
residual = hidden_states
hidden_states = self.norm0(hidden_states)
attn_out = self.attention_qkvpacked(
hidden_states, cu_seqlens, rope_freqs_cis=rope_freqs_cis
hidden_states,
cu_seqlens,
rope_freqs_cis=rope_freqs_cis,
max_seqlen=max_seqlen,
)
hidden_states = residual + attn_out
@@ -478,22 +533,39 @@ class MoonVitEncoder(nn.Module):
)
self.final_layernorm = nn.LayerNorm(hidden_dim)
def forward(
self, hidden_states: torch.Tensor, grid_hw: torch.Tensor
def get_rope_freqs_cis(
self,
grid_hws_list: list[list[int]] | list[tuple[int, int]],
) -> torch.Tensor:
rope_freqs_cis = self.rope_2d.get_freqs_cis_by_seqlens(grid_hws=grid_hw)
return self.rope_2d.get_freqs_cis_by_seqlens_list(grid_hws_list)
lengths = torch.cat(
(
torch.zeros(1, device=hidden_states.device, dtype=grid_hw.dtype),
(grid_hw[:, 0] * grid_hw[:, 1]).to(hidden_states.device),
def forward(
self,
hidden_states: torch.Tensor,
grid_hw: torch.Tensor | None = None,
*,
cu_seqlens: torch.Tensor | None = None,
rope_freqs_cis: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor:
if rope_freqs_cis is None:
rope_freqs_cis = self.rope_2d.get_freqs_cis_by_seqlens(grid_hws=grid_hw)
if cu_seqlens is None:
lengths = torch.cat(
(
torch.zeros(1, device=hidden_states.device, dtype=grid_hw.dtype),
(grid_hw[:, 0] * grid_hw[:, 1]).to(hidden_states.device),
)
)
)
cu_seqlens = lengths.cumsum(dim=0, dtype=torch.int32)
cu_seqlens = lengths.cumsum(dim=0, dtype=torch.int32)
for _, block in enumerate(self.blocks):
hidden_states = block(
hidden_states, cu_seqlens, rope_freqs_cis=rope_freqs_cis
hidden_states,
cu_seqlens,
rope_freqs_cis=rope_freqs_cis,
max_seqlen=max_seqlen,
)
hidden_states = self.final_layernorm(hidden_states)
@@ -530,6 +602,54 @@ def patch_merger(
return outputs
def patch_merger_packed(
x: torch.Tensor,
gather_idx: torch.Tensor,
merge_kernel_size: tuple[int, int],
) -> torch.Tensor:
"""CUDA-graph-safe equivalent of :func:`patch_merger`.
Uses a precomputed index tensor to gather the per-token reshape +
permute that ``patch_merger`` does inside a Python loop. The output is
the concatenated 3D tensor ``(sum(new_h * new_w), kh * kw, d_model)``,
matching what ``torch.cat(patch_merger(...))`` would produce.
"""
kh, kw = merge_kernel_size
d_model = x.size(-1)
return x.index_select(0, gather_idx).view(-1, kh * kw, d_model)
def _build_merge_gather_idx(
grid_hws_list: list[list[int]] | list[tuple[int, int]],
merge_kernel_size: tuple[int, int],
) -> np.ndarray:
"""Build the per-token gather indices used by :func:`patch_merger_packed`.
For each item with grid (h, w) and merge kernel (kh, kw), the output
block at position (nh, nw) gathers the kh*kw input tokens at rows
(nh*kh + ih, nw*kw + iw) of that item, in (ih, iw) row-major order.
"""
kh, kw = merge_kernel_size
parts: list[np.ndarray] = []
pre_sum = 0
for h, w in grid_hws_list:
new_h, new_w = h // kh, w // kw
nh = np.arange(new_h, dtype=np.int64).reshape(new_h, 1, 1, 1)
nw = np.arange(new_w, dtype=np.int64).reshape(1, new_w, 1, 1)
ih = np.arange(kh, dtype=np.int64).reshape(1, 1, kh, 1)
iw = np.arange(kw, dtype=np.int64).reshape(1, 1, 1, kw)
# Linearized input row = (nh*kh + ih) * w + (nw*kw + iw), offset by
# the per-item base ``pre_sum``. Output is laid out as
# (new_h, new_w, kh, kw) which patch_merger flattens to
# (new_h*new_w, kh*kw).
idx = pre_sum + (nh * kh + ih) * w + (nw * kw + iw)
parts.append(idx.reshape(-1))
pre_sum += h * w
if not parts:
return np.zeros(0, dtype=np.int64)
return np.concatenate(parts)
class MoonVitPretrainedModel(PreTrainedModel):
config_class = MoonViTConfig
model_type = "moonvit"
@@ -570,17 +690,126 @@ class MoonVitPretrainedModel(PreTrainedModel):
prefix=f"{prefix}.encoder",
)
def prepare_encoder_metadata(
self,
grid_hws_list: list[list[int]] | list[tuple[int, int]],
*,
max_batch_size: int | None = None,
max_seqlen_override: int | None = None,
device: torch.device | None = None,
) -> dict[str, Any]:
"""Precompute every grid-dependent input the encoder needs.
Used by the CUDA graph capture and replay paths to precompute
every grid-dependent input outside the captured graph, so per-grid
Python iteration and ``.tolist()`` round-trips are fine; the
values are then copied into fixed-shape buffers for replay.
Args:
grid_hws_list: List of ``(h, w)`` patch-grid sizes per image.
max_batch_size: When set, ``cu_seqlens`` is right-padded with
its last value so the buffer covers up to this many
sequences. Required at CUDA graph capture/replay so the
buffer shape matches what was recorded; padding entries
are zero-length sequences and are ignored by varlen
attention.
max_seqlen_override: Override the per-replay max sequence
length scalar. At capture this must be a safe upper bound
(worst case: a single image consuming the full token
budget) because the value is baked into the captured
graph.
device: Device for the metadata tensors. Defaults to the
model's parameter device.
"""
if device is None:
device = next(self.parameters()).device
# Normalize to a list of plain Python int pairs so the helpers
# below never need ``.tolist()`` on a tensor.
grid_pairs: list[tuple[int, int]] = [(int(h), int(w)) for h, w in grid_hws_list]
metadata: dict[str, Any] = {}
pos_embeds = self.patch_embed.pos_emb.get_pos_embeds(grid_pairs)
metadata["pos_embeds"] = pos_embeds.to(device=device)
rope_freqs_cis = self.encoder.get_rope_freqs_cis(grid_pairs)
metadata["rope_freqs_cis"] = rope_freqs_cis.to(device=device)
grid_arr = np.array(grid_pairs, dtype=np.int64)
seq_lens = (grid_arr[:, 0] * grid_arr[:, 1]).astype(np.int32)
cu_seqlens_np = np.concatenate(
[
np.zeros(1, dtype=np.int32),
seq_lens.cumsum(dtype=np.int32),
]
)
if max_batch_size is not None:
num_seqs = len(cu_seqlens_np) - 1
if num_seqs < max_batch_size:
cu_seqlens_np = np.concatenate(
[
cu_seqlens_np,
np.full(
max_batch_size - num_seqs,
cu_seqlens_np[-1],
dtype=np.int32,
),
]
)
metadata["cu_seqlens"] = torch.from_numpy(cu_seqlens_np).to(device)
if max_seqlen_override is not None:
max_seqlen_val = int(max_seqlen_override)
else:
max_seqlen_val = int(seq_lens.max()) if len(seq_lens) > 0 else 0
# Keep on CPU: attention wrappers may call .item() on this scalar
# and we want that materialization to happen outside the captured
# graph (the value is constant per capture anyway).
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
gather_idx_np = _build_merge_gather_idx(grid_pairs, self.merge_kernel_size)
metadata["merge_gather_idx"] = torch.from_numpy(gather_idx_np).to(device)
return metadata
def forward(
self, pixel_values: torch.Tensor, grid_hw: torch.Tensor
) -> torch.Tensor:
self,
pixel_values: torch.Tensor,
grid_hw: torch.Tensor,
*,
encoder_metadata: dict[str, Any] | None = None,
) -> torch.Tensor | list[torch.Tensor]:
"""
Args:
pixel_values (torch.Tensor): The input pixel values.
grid_hw (torch.Tensor): The grid height and width.
Returns:
torch.Tensor: The output tokens.
encoder_metadata: Optional precomputed metadata produced by
:meth:`prepare_encoder_metadata`. When provided every
``.tolist()`` call in the forward path is skipped, the
returned tensor is the packed
``(sum(new_h*new_w), kh*kw, hidden_size)`` form (suitable
for CUDA graph capture/replay), and ``grid_hw`` is unused.
When ``None`` the legacy path runs and returns a list of
per-image tensors.
"""
if encoder_metadata is not None:
hidden_states = self.patch_embed(
pixel_values, pos_embeds=encoder_metadata["pos_embeds"]
)
hidden_states = self.encoder(
hidden_states,
cu_seqlens=encoder_metadata["cu_seqlens"],
rope_freqs_cis=encoder_metadata["rope_freqs_cis"],
max_seqlen=encoder_metadata["max_seqlen"],
)
return patch_merger_packed(
hidden_states,
encoder_metadata["merge_gather_idx"],
merge_kernel_size=self.merge_kernel_size,
)
hidden_states = self.patch_embed(pixel_values, grid_hw)
hidden_states = self.encoder(hidden_states, grid_hw)
hidden_states = patch_merger(