[Minimax-M3][Perf] Use triton_mrope for vision tower + int64 offset fix for triton_mrope (#58526)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Kimi <noreply@moonshot.cn>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Thien Tran
2026-09-24 18:30:32 -07:00
committed by GitHub
co-authored by Kimi Isotr0py
parent ade1056ee5
commit 16070ed7e9
4 changed files with 279 additions and 178 deletions
+73 -1
View File
@@ -6,7 +6,10 @@ import pytest
import torch
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.mrope import apply_interleaved_rope
from vllm.model_executor.layers.rotary_embedding.mrope import (
apply_interleaved_rope,
triton_mrope,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.config import get_config
from vllm.utils.torch_utils import set_random_seed
@@ -174,6 +177,75 @@ def test_mrope(
torch.testing.assert_close(query_native, query_cuda, atol=atol, rtol=rtol)
torch.testing.assert_close(key_native, key_cuda, atol=atol, rtol=rtol)
# Strided q/k views of a packed buffer (the MiniMax M3 ViT usage):
# bitwise-identical to the contiguous run, neighboring columns untouched.
pad = 16
packed = torch.zeros(
num_tokens,
query.shape[1] + key.shape[1] + pad,
dtype=dtype,
device=device,
)
packed[:, : query.shape[1]] = query
packed[:, query.shape[1] : query.shape[1] + key.shape[1]] = key
sentinel = packed[:, -pad:].clone()
q_strided = packed[:, : query.shape[1]]
k_strided = packed[:, query.shape[1] : query.shape[1] + key.shape[1]]
mrope_helper_class.forward_cuda(positions, q_strided, k_strided)
torch.testing.assert_close(q_strided, query_cuda, atol=0, rtol=0)
torch.testing.assert_close(k_strided, key_cuda, atol=0, rtol=0)
torch.testing.assert_close(packed[:, -pad:], sentinel, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
)
def test_mrope_row_offset_int64():
# MiniMax M3 ViT worst case: a max-size video item packs ~768k patches,
# and num_tokens * packed-qkv row stride (3840) exceeds 2^31 at 559,241
# rows. Row-base math must be int64 or those rows wrap to negative
# offsets (illegal memory access at startup profiling).
set_random_seed(0)
n_qh, n_kh, hd, rd = 16, 16, 80, 78
mrope_section = [13, 13, 13]
row_stride = (n_qh + n_kh + n_kh) * hd # packed qkv row
num_tokens = 2**31 // row_stride + 8
free, _ = current_platform.mem_get_info()
if free < 2 * num_tokens * row_stride * 2:
pytest.skip("not enough free device memory")
packed = torch.empty(num_tokens, row_stride, dtype=torch.bfloat16, device=device)
q_fill = torch.randn(n_qh * hd, dtype=torch.bfloat16, device=device)
k_fill = torch.randn(n_kh * hd, dtype=torch.bfloat16, device=device)
packed[:, : n_qh * hd] = q_fill
packed[:, n_qh * hd : (n_qh + n_kh) * hd] = k_fill
half_rd = rd // 2
cos = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device)
sin = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device)
q = packed[:, : n_qh * hd]
k = packed[:, n_qh * hd : (n_qh + n_kh) * hd]
triton_mrope(q, k, cos, sin, mrope_section, hd, rd, False, True)
t_end = mrope_section[0]
h_end = t_end + mrope_section[1]
for r in (0, num_tokens - 8, num_tokens - 1):
cos_row = torch.cat(
[cos[0, r, :t_end], cos[1, r, t_end:h_end], cos[2, r, h_end:]]
)
sin_row = torch.cat(
[sin[0, r, :t_end], sin[1, r, t_end:h_end], sin[2, r, h_end:]]
)
for fill, out in ((q_fill, q[r]), (k_fill, k[r])):
row = fill.view(-1, hd).float()
r1, r2 = row[:, :half_rd], row[:, half_rd:rd]
exp = torch.cat(
[r1 * cos_row - r2 * sin_row, r2 * cos_row + r1 * sin_row, row[:, rd:]],
dim=1,
)
torch.testing.assert_close(
out, exp.view(-1).to(torch.bfloat16), atol=1e-2, rtol=1.6e-2
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
@@ -20,6 +20,8 @@ def _triton_mrope_forward(
cos,
sin,
num_tokens,
q_token_stride,
k_token_stride,
n_qh: tl.constexpr,
n_kh: tl.constexpr,
hd: tl.constexpr,
@@ -39,9 +41,10 @@ def _triton_mrope_forward(
# and supports cos and sin cache with shape (3, num_tokens, rotary_dim // 2)
# instead of (3, bsz, seq_len, head_dim), also supports interleaved rotary
pid = tl.program_id(0)
# locate start address
q_ptr = q_ptr + pid * (n_qh * hd)
k_ptr = k_ptr + pid * (n_kh * hd)
# locate start address (int64: num_tokens * row_stride exceeds 2^31 for
# worst-case ViT forwards, e.g. MiniMax M3 packs ~768k patches/item)
q_ptr = q_ptr + pid.to(tl.int64) * q_token_stride
k_ptr = k_ptr + pid.to(tl.int64) * k_token_stride
# ####################################################################
# get the cos(mθ_{i...d/2}) and sin(mθ_{i...d/2}) for token position
@@ -186,6 +189,10 @@ def triton_mrope(
adjacent (GPT-J) layout.
"""
# The kernel indexes each token row through an explicit row stride and
# requires dense rows.
assert q.dim() == 2 and q.stride(-1) == 1
assert k.dim() == 2 and k.stride(-1) == 1
n_row, n_q_head_head_dim = q.shape
n_q_head = n_q_head_head_dim // head_size
n_kv_head = k.shape[1] // head_size
@@ -193,10 +200,6 @@ def triton_mrope(
pad_n_q_head = triton.next_power_of_2(n_q_head)
pad_n_kv_head = triton.next_power_of_2(n_kv_head)
# ensure tensors passed into the kernel are contiguous.
# It will be no-op if they are already contiguous
q = q.contiguous()
k = k.contiguous()
cos = cos.contiguous()
sin = sin.contiguous()
@@ -211,6 +214,8 @@ def triton_mrope(
cos,
sin,
n_row,
q.stride(0),
k.stride(0),
n_q_head,
n_kv_head,
head_size,
@@ -0,0 +1,153 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr
# is an offset slice of the packed H2D buffer, so its 16B alignment varies
# with grid count; without these the kernel recompiles mid-serving.
@triton.jit(
do_not_specialize=["num_tokens", "num_grids"],
do_not_specialize_on_alignment=["grids_ptr"],
)
def _vit_mrope_setup_kernel(
inv_freq_t_ptr,
inv_freq_h_ptr,
inv_freq_w_ptr,
cos_out_ptr,
sin_out_ptr,
cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid
grids_ptr, # (G, 3) int32: t, h, w per grid
num_tokens,
num_grids,
half_t: tl.constexpr,
half_h: tl.constexpr,
half_w: tl.constexpr,
spatial_merge: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one
launch: compute each token's (t, h, w) position arithmetically and the
rotation in fp32 from the per-axis inv_freq. Only each axis's mrope
section columns are written; the rest is left uninitialized (the
consumer's per-axis masked loads read only the section columns)."""
pid = tl.program_id(0)
tok = pid * BLOCK_N + tl.arange(0, BLOCK_N)
tok_mask = tok < num_tokens
# Grid (segment) index per token; cu_seqlens is nondecreasing.
seg = tl.zeros([BLOCK_N], dtype=tl.int32)
for g in range(num_grids):
end = tl.load(cu_seqlens_ptr + g + 1)
seg += (tok >= end).to(tl.int32)
seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0)
grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1)
grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1)
local = tok - seg_start
rem = local % (grid_h * grid_w)
tpos = local // (grid_h * grid_w)
# Within a frame tokens are ordered by spatial-merge block (bh, bw, mh,
# mw), matching the reference ViT position-id permute.
bh = rem // (spatial_merge * grid_w)
r = rem % (spatial_merge * grid_w)
bw = r // (spatial_merge * spatial_merge)
r2 = r % (spatial_merge * spatial_merge)
hpos = bh * spatial_merge + r2 // spatial_merge
wpos = bw * spatial_merge + r2 % spatial_merge
d = tl.arange(0, BLOCK_D)
half_rot = half_t + half_h + half_w
row = (tok * half_rot)[:, None] + d[None, :]
plane = num_tokens * half_rot
# Per-axis compute+store to cap live registers; the fp32 trig sequence
# wants few elements per thread (hence small BLOCK_N, many warps).
in_t = d < half_t
inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0)
freq = tpos.to(tl.float32)[:, None] * inv_t[None, :]
mask = tok_mask[:, None] & in_t[None, :]
tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask)
in_h = (d >= half_t) & (d < half_t + half_h)
inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0)
freq = hpos.to(tl.float32)[:, None] * inv_h[None, :]
mask = tok_mask[:, None] & in_h[None, :]
tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask)
in_w = (d >= half_t + half_h) & (d < half_rot)
inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0)
freq = wpos.to(tl.float32)[:, None] * inv_w[None, :]
mask = tok_mask[:, None] & in_w[None, :]
tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask)
tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask)
def vit_mrope_setup(
inv_freq_t: torch.Tensor,
inv_freq_h: torch.Tensor,
inv_freq_w: torch.Tensor,
grid_thw: list[list[int]],
spatial_merge_size: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope.
One fused kernel: positions are computed arithmetically from the grids
and the rotation comes from the per-axis inv_freq, so there is no
cos/sin cache to size or grow.
Args:
inv_freq_t: t-axis inverse frequencies, (t_dim // 2,).
inv_freq_h: h-axis inverse frequencies, (h_dim // 2,).
inv_freq_w: w-axis inverse frequencies, (w_dim // 2,).
grid_thw: per-grid [t, h, w] patch counts (host metadata).
spatial_merge_size: spatial merge factor of the ViT.
"""
device = inv_freq_t.device
cu = [0]
for t, h, w in grid_thw:
cu.append(cu[-1] + t * h * w)
n = cu[-1]
g = len(grid_thw)
# grid_thw is per-request host metadata, so one H2D is unavoidable;
# pack cu_seqlens and grids into a single pinned async copy.
buf = async_tensor_h2d(
cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32
)
cu_seqlens = buf[: g + 1]
grids = buf[g + 1 :].view(g, 3)
half_t = inv_freq_t.numel()
half_h = inv_freq_h.numel()
half_w = inv_freq_w.numel()
half_rot = half_t + half_h + half_w
cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
_vit_mrope_setup_kernel[(triton.cdiv(n, 32),)](
inv_freq_t,
inv_freq_h,
inv_freq_w,
cos,
sin,
cu_seqlens,
grids,
n,
g,
half_t=half_t,
half_h=half_h,
half_w=half_w,
spatial_merge=spatial_merge_size,
BLOCK_N=32,
BLOCK_D=triton.next_power_of_2(half_rot),
num_warps=8,
)
return cos, sin
+41 -170
View File
@@ -21,20 +21,16 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
from vllm.model_executor.layers.rotary_embedding.mrope_vit_setup import (
vit_mrope_setup,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.vision import (
get_vit_attn_backend,
is_vit_use_data_parallel,
)
from vllm.platforms import current_platform
# ROCm caps a kernel-launch gridDim.y at 65536. The HIP flash-attn Triton
# rotary kernel launches grid.y = cdiv(seqlen, BLOCK_M), so it fails with
# hipErrorInvalidValue once cdiv(seqlen, BLOCK_M) > 65536. Used below to decide
# when RoPE must be applied per video segment instead of in one launch.
_HIP_MAX_GRID_DIM_Y = 65536
class MiniMaxVLPatchEmbed(nn.Module):
@@ -125,56 +121,13 @@ class MiniMaxVLAttention(nn.Module):
head_size=self.head_dim,
prefix=f"{prefix}.attn",
)
# ApplyRotaryEmb handles the internal cos/sin repeat and partial
# rotation (ro_dim = half_rot_dim * 2 < head_dim for MiniMax).
# enable_fp32_compute=True runs the rotation in fp32 (q/k upcast,
# fp32 cos/sin), matching the reference ``_minimax_rope_applier``.
self.apply_rotary_emb = ApplyRotaryEmb(
enforce_enable=True, enable_fp32_compute=True
)
def _apply_rotary_emb(
self,
qk_reshaped: torch.Tensor,
rotary_cos: torch.Tensor,
rotary_sin: torch.Tensor,
seq_len: int,
rotary_segment_lengths: list[int] | None,
) -> torch.Tensor:
# Default fast path (all NVIDIA inputs, and ROCm short clips/images):
# a single rotary kernel launch. ``rotary_segment_lengths`` is only
# populated on ROCm (see ``MiniMaxVLVisionTransformer.forward``), so
# the per-segment path below is ROCm-only and never touches the
# NVIDIA/CUDA code path.
if not current_platform.is_rocm() or rotary_segment_lengths is None:
return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin)
# ROCm only: the HIP flash-attn Triton rotary kernel fails with
# hipErrorInvalidValue once grid.y = cdiv(seqlen, BLOCK_M) exceeds
# _HIP_MAX_GRID_DIM_Y (65536). BLOCK_M is 8 for rotary_dim <= 128
# (MiniMax-M3 vision: rotary_dim=78), giving a hard limit of
# 65536 * BLOCK_M tokens — measured exactly as 524288 OK / 524289 fail.
# Only long videos cross it; since vision_segment_max_frames caps each
# segment at a few frames (<< limit), applying RoPE per segment keeps
# every sub-call in range. Splitting on segment boundaries is
# mathematically exact because rotary_cos/sin are precomputed per token.
# Images and short clips stay on the single-kernel fast path above.
rotary_dim = rotary_cos.shape[-1] * 2
block_m = 8 if rotary_dim <= 128 else 4
hip_rotary_max_seqlen = _HIP_MAX_GRID_DIM_Y * block_m
if seq_len <= hip_rotary_max_seqlen or len(rotary_segment_lengths) <= 1:
return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin)
qk_segments = qk_reshaped.split(rotary_segment_lengths, dim=1)
cos_segments = rotary_cos.split(rotary_segment_lengths, dim=0)
sin_segments = rotary_sin.split(rotary_segment_lengths, dim=0)
return torch.cat(
[
self.apply_rotary_emb(qk_s, cos_s, sin_s)
for qk_s, cos_s, sin_s in zip(qk_segments, cos_segments, sin_segments)
],
dim=1,
)
# Partial 3D RoPE geometry: rot_dim is split evenly across t/h/w
# (same formula as MiniMaxVLVisionTransformer; rot_dim may be <
# head_dim). mrope_section is in half-dim units for triton_mrope.
rope_dims = 2 * (self.head_dim // 2)
axis_dim = 2 * ((rope_dims // 3) // 2)
self.rotary_dim = 3 * axis_dim
self.mrope_section = [axis_dim // 2] * 3
def forward(
self,
@@ -183,33 +136,34 @@ class MiniMaxVLAttention(nn.Module):
rotary_cos: torch.Tensor,
rotary_sin: torch.Tensor,
max_seqlen: torch.Tensor,
rotary_segment_lengths: list[int] | None = None,
sequence_lengths: torch.Tensor | None = None,
) -> torch.Tensor:
# x: (N, 1, embed_dim) [seq=N, batch=1, chan=embed_dim]
x_qkv, _ = self.qkv_proj(x) # (N, 1, 3 * heads_per_part * head_dim)
seq_len, batch_size, _ = x_qkv.shape
# Rearrange to (b=1, N, 3, heads, head_dim) — same as Qwen2_5_VisionAttention
qkv = rearrange(
x_qkv,
"s b (three head d) -> b s three head d",
three=3,
head=self.num_heads_per_partition,
)
qk, v = qkv[:, :, :2], qkv[:, :, 2] # (b,N,2,h,d) and (b,N,h,d)
qkv = x_qkv.view(seq_len, 3, self.num_heads_per_partition, self.head_dim)
v = qkv[:, 2].unsqueeze(0) # (b=1, N, heads, head_dim) strided view
# Stack q/k → (2*b, N, heads, head_dim) for joint RoPE application.
# rotary_cos/sin: (N, half_rot_dim) — ApplyRotaryEmb expands internally
# and rotates only the first 2*half_rot_dim dims, passing the rest through.
qk_reshaped = rearrange(qk, "b s two h d -> (two b) s h d", two=2).contiguous()
qk_rotated = self._apply_rotary_emb(
qk_reshaped, rotary_cos, rotary_sin, seq_len, rotary_segment_lengths
# In-place partial 3D RoPE on strided q/k views of the packed qkv
# tensor: one kernel launch, no repack/upcast copies. fp32 cos/sin
# make the kernel compute in fp32, matching the reference
# ``_minimax_rope_applier`` precision.
q = qkv[:, 0].reshape(seq_len, -1)
k = qkv[:, 1].reshape(seq_len, -1)
triton_mrope(
q,
k,
rotary_cos,
rotary_sin,
self.mrope_section,
self.head_dim,
self.rotary_dim,
mrope_interleaved=False,
is_neox_style=True,
)
qk_rotated = qk_rotated.view(
2, batch_size, seq_len, self.num_heads_per_partition, self.head_dim
)
q, k = qk_rotated.unbind(dim=0) # each (b=1, N, heads, head_dim)
q = q.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim)
k = k.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim)
# Flash attention → (b, N, heads, head_dim)
context = self.attn(
@@ -272,7 +226,6 @@ class MiniMaxVLEncoderLayer(nn.Module):
rotary_cos: torch.Tensor,
rotary_sin: torch.Tensor,
max_seqlen: torch.Tensor,
rotary_segment_lengths: list[int] | None = None,
sequence_lengths: torch.Tensor | None = None,
) -> torch.Tensor:
# x: (N, 1, hidden_size)
@@ -282,7 +235,6 @@ class MiniMaxVLEncoderLayer(nn.Module):
rotary_cos,
rotary_sin,
max_seqlen,
rotary_segment_lengths,
sequence_lengths,
)
residual = x
@@ -324,7 +276,6 @@ class MiniMaxVLEncoder(nn.Module):
rotary_cos: torch.Tensor,
rotary_sin: torch.Tensor,
max_seqlen: torch.Tensor,
rotary_segment_lengths: list[int] | None = None,
sequence_lengths: torch.Tensor | None = None,
) -> torch.Tensor:
for layer in self.layers:
@@ -334,7 +285,6 @@ class MiniMaxVLEncoder(nn.Module):
rotary_cos,
rotary_sin,
max_seqlen,
rotary_segment_lengths,
sequence_lengths,
)
return x
@@ -427,82 +377,6 @@ class MiniMaxVLVisionTransformer(nn.Module):
# out_hidden_size needed by run_dp_sharded_mrope_vision_model
self.out_hidden_size = embed_dim
# ── RoPE helpers ─────────────────────────────────────────────────────
def _get_3d_rope_embed(
self, grid_t: int, grid_h: int, grid_w: int, spatial_merge_size: int
) -> torch.Tensor:
"""Compute 3D RoPE frequencies for a single (T, H, W) grid.
Returns (T*H*W, half_rot_dim) on the same device as inv_freq buffers.
Mirrors the reference ``_get_3d_rope_embed`` exactly.
"""
tokens_per_frame = grid_h * grid_w
tpos_ids = (
torch.arange(grid_t, device=self.inv_freq_t.device)
.unsqueeze(1)
.expand(-1, tokens_per_frame)
.flatten()
)
hpos_ids = (
torch.arange(grid_h, device=self.inv_freq_h.device)
.unsqueeze(1)
.expand(-1, grid_w)
.reshape(
grid_h // spatial_merge_size,
spatial_merge_size,
grid_w // spatial_merge_size,
spatial_merge_size,
)
.permute(0, 2, 1, 3)
.unsqueeze(0)
.expand(grid_t, -1, -1, -1, -1)
.flatten()
)
wpos_ids = (
torch.arange(grid_w, device=self.inv_freq_w.device)
.unsqueeze(0)
.expand(grid_h, -1)
.reshape(
grid_h // spatial_merge_size,
spatial_merge_size,
grid_w // spatial_merge_size,
spatial_merge_size,
)
.permute(0, 2, 1, 3)
.unsqueeze(0)
.expand(grid_t, -1, -1, -1, -1)
.flatten()
)
max_t = max(grid_t, 1)
max_hw = max(grid_h, grid_w)
seq_t = torch.arange(
max_t, device=self.inv_freq_t.device, dtype=self.inv_freq_t.dtype
)
seq_hw = torch.arange(
max_hw, device=self.inv_freq_h.device, dtype=self.inv_freq_h.dtype
)
freqs_t = torch.outer(seq_t, self.inv_freq_t) # (max_t, t_dim/2)
freqs_h = torch.outer(seq_hw, self.inv_freq_h) # (max_hw, h_dim/2)
freqs_w = torch.outer(seq_hw, self.inv_freq_w) # (max_hw, w_dim/2)
return torch.cat(
[freqs_t[tpos_ids], freqs_h[hpos_ids], freqs_w[wpos_ids]], dim=-1
) # (T*H*W, half_rot_dim)
def _get_rope_embed_3d(
self, grid_thw: list[list[int]], spatial_merge_size: int
) -> torch.Tensor:
embeds = [
self._get_3d_rope_embed(t, h, w, spatial_merge_size) for t, h, w in grid_thw
]
return torch.cat(embeds, dim=0) # (total_N, half_rot_dim)
# ── Frame-limit helper (mirrors the reference) ───────────────────────
def _apply_max_frames_limit(self, grid_thw: list[list[int]]) -> list[list[int]]:
@@ -557,21 +431,19 @@ class MiniMaxVLVisionTransformer(nn.Module):
hidden.device,
)
# 3D RoPE: (total_N, half_rot_dim); ApplyRotaryEmb expands internally
freqs = self._get_rope_embed_3d(limited, self.spatial_merge_size)
freqs = freqs.to(device=hidden.device)
# Keep cos/sin in fp32; ApplyRotaryEmb(enable_fp32_compute=True) runs the
# rotation in fp32 to match the reference precision.
rotary_cos, rotary_sin = freqs.cos(), freqs.sin()
# 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for
# triton_mrope from one fused setup kernel. Kept fp32 so the kernel
# computes the rotation in fp32, matching the reference.
rotary_cos, rotary_sin = vit_mrope_setup(
self.inv_freq_t,
self.inv_freq_h,
self.inv_freq_w,
limited,
self.spatial_merge_size,
)
# Encoder expects (N, 1, hidden_size) — add batch dim
hidden = hidden.unsqueeze(1)
# On ROCm, the flash_attn Triton rotary kernel can fail with
# hipErrorInvalidValue when seqlen is very large, e.g. 192k video
# tokens; pass per-segment lengths so RoPE can be applied in chunks.
# On other platforms leave it None -> single-kernel fast path, so the
# NVIDIA/CUDA code path is unchanged.
rotary_segment_lengths = lens if current_platform.is_rocm() else None
hidden = self.encoder(
hidden,
@@ -579,7 +451,6 @@ class MiniMaxVLVisionTransformer(nn.Module):
rotary_cos,
rotary_sin,
max_seqlen,
rotary_segment_lengths,
sequence_lengths=sequence_lengths,
)
hidden = hidden.squeeze(1) # back to (total_N, hidden_size)