[Minimax-M3][Perf] Use triton_mrope for vision tower + int64 offset fix for triton_mrope (#58526)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Kimi <noreply@moonshot.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Kimi
Isotr0py
parent
ade1056ee5
commit
16070ed7e9
@@ -6,7 +6,10 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import apply_interleaved_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import (
|
||||
apply_interleaved_rope,
|
||||
triton_mrope,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import get_config
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -174,6 +177,75 @@ def test_mrope(
|
||||
torch.testing.assert_close(query_native, query_cuda, atol=atol, rtol=rtol)
|
||||
torch.testing.assert_close(key_native, key_cuda, atol=atol, rtol=rtol)
|
||||
|
||||
# Strided q/k views of a packed buffer (the MiniMax M3 ViT usage):
|
||||
# bitwise-identical to the contiguous run, neighboring columns untouched.
|
||||
pad = 16
|
||||
packed = torch.zeros(
|
||||
num_tokens,
|
||||
query.shape[1] + key.shape[1] + pad,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
packed[:, : query.shape[1]] = query
|
||||
packed[:, query.shape[1] : query.shape[1] + key.shape[1]] = key
|
||||
sentinel = packed[:, -pad:].clone()
|
||||
q_strided = packed[:, : query.shape[1]]
|
||||
k_strided = packed[:, query.shape[1] : query.shape[1] + key.shape[1]]
|
||||
mrope_helper_class.forward_cuda(positions, q_strided, k_strided)
|
||||
torch.testing.assert_close(q_strided, query_cuda, atol=0, rtol=0)
|
||||
torch.testing.assert_close(k_strided, key_cuda, atol=0, rtol=0)
|
||||
torch.testing.assert_close(packed[:, -pad:], sentinel, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
|
||||
)
|
||||
def test_mrope_row_offset_int64():
|
||||
# MiniMax M3 ViT worst case: a max-size video item packs ~768k patches,
|
||||
# and num_tokens * packed-qkv row stride (3840) exceeds 2^31 at 559,241
|
||||
# rows. Row-base math must be int64 or those rows wrap to negative
|
||||
# offsets (illegal memory access at startup profiling).
|
||||
set_random_seed(0)
|
||||
n_qh, n_kh, hd, rd = 16, 16, 80, 78
|
||||
mrope_section = [13, 13, 13]
|
||||
row_stride = (n_qh + n_kh + n_kh) * hd # packed qkv row
|
||||
num_tokens = 2**31 // row_stride + 8
|
||||
free, _ = current_platform.mem_get_info()
|
||||
if free < 2 * num_tokens * row_stride * 2:
|
||||
pytest.skip("not enough free device memory")
|
||||
|
||||
packed = torch.empty(num_tokens, row_stride, dtype=torch.bfloat16, device=device)
|
||||
q_fill = torch.randn(n_qh * hd, dtype=torch.bfloat16, device=device)
|
||||
k_fill = torch.randn(n_kh * hd, dtype=torch.bfloat16, device=device)
|
||||
packed[:, : n_qh * hd] = q_fill
|
||||
packed[:, n_qh * hd : (n_qh + n_kh) * hd] = k_fill
|
||||
half_rd = rd // 2
|
||||
cos = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device)
|
||||
sin = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device)
|
||||
q = packed[:, : n_qh * hd]
|
||||
k = packed[:, n_qh * hd : (n_qh + n_kh) * hd]
|
||||
triton_mrope(q, k, cos, sin, mrope_section, hd, rd, False, True)
|
||||
|
||||
t_end = mrope_section[0]
|
||||
h_end = t_end + mrope_section[1]
|
||||
for r in (0, num_tokens - 8, num_tokens - 1):
|
||||
cos_row = torch.cat(
|
||||
[cos[0, r, :t_end], cos[1, r, t_end:h_end], cos[2, r, h_end:]]
|
||||
)
|
||||
sin_row = torch.cat(
|
||||
[sin[0, r, :t_end], sin[1, r, t_end:h_end], sin[2, r, h_end:]]
|
||||
)
|
||||
for fill, out in ((q_fill, q[r]), (k_fill, k[r])):
|
||||
row = fill.view(-1, hd).float()
|
||||
r1, r2 = row[:, :half_rd], row[:, half_rd:rd]
|
||||
exp = torch.cat(
|
||||
[r1 * cos_row - r2 * sin_row, r2 * cos_row + r1 * sin_row, row[:, rd:]],
|
||||
dim=1,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
out, exp.view(-1).to(torch.bfloat16), atol=1e-2, rtol=1.6e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
|
||||
|
||||
@@ -20,6 +20,8 @@ def _triton_mrope_forward(
|
||||
cos,
|
||||
sin,
|
||||
num_tokens,
|
||||
q_token_stride,
|
||||
k_token_stride,
|
||||
n_qh: tl.constexpr,
|
||||
n_kh: tl.constexpr,
|
||||
hd: tl.constexpr,
|
||||
@@ -39,9 +41,10 @@ def _triton_mrope_forward(
|
||||
# and supports cos and sin cache with shape (3, num_tokens, rotary_dim // 2)
|
||||
# instead of (3, bsz, seq_len, head_dim), also supports interleaved rotary
|
||||
pid = tl.program_id(0)
|
||||
# locate start address
|
||||
q_ptr = q_ptr + pid * (n_qh * hd)
|
||||
k_ptr = k_ptr + pid * (n_kh * hd)
|
||||
# locate start address (int64: num_tokens * row_stride exceeds 2^31 for
|
||||
# worst-case ViT forwards, e.g. MiniMax M3 packs ~768k patches/item)
|
||||
q_ptr = q_ptr + pid.to(tl.int64) * q_token_stride
|
||||
k_ptr = k_ptr + pid.to(tl.int64) * k_token_stride
|
||||
|
||||
# ####################################################################
|
||||
# get the cos(mθ_{i...d/2}) and sin(mθ_{i...d/2}) for token position
|
||||
@@ -186,6 +189,10 @@ def triton_mrope(
|
||||
adjacent (GPT-J) layout.
|
||||
|
||||
"""
|
||||
# The kernel indexes each token row through an explicit row stride and
|
||||
# requires dense rows.
|
||||
assert q.dim() == 2 and q.stride(-1) == 1
|
||||
assert k.dim() == 2 and k.stride(-1) == 1
|
||||
n_row, n_q_head_head_dim = q.shape
|
||||
n_q_head = n_q_head_head_dim // head_size
|
||||
n_kv_head = k.shape[1] // head_size
|
||||
@@ -193,10 +200,6 @@ def triton_mrope(
|
||||
pad_n_q_head = triton.next_power_of_2(n_q_head)
|
||||
pad_n_kv_head = triton.next_power_of_2(n_kv_head)
|
||||
|
||||
# ensure tensors passed into the kernel are contiguous.
|
||||
# It will be no-op if they are already contiguous
|
||||
q = q.contiguous()
|
||||
k = k.contiguous()
|
||||
cos = cos.contiguous()
|
||||
sin = sin.contiguous()
|
||||
|
||||
@@ -211,6 +214,8 @@ def triton_mrope(
|
||||
cos,
|
||||
sin,
|
||||
n_row,
|
||||
q.stride(0),
|
||||
k.stride(0),
|
||||
n_q_head,
|
||||
n_kv_head,
|
||||
head_size,
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr
|
||||
# is an offset slice of the packed H2D buffer, so its 16B alignment varies
|
||||
# with grid count; without these the kernel recompiles mid-serving.
|
||||
@triton.jit(
|
||||
do_not_specialize=["num_tokens", "num_grids"],
|
||||
do_not_specialize_on_alignment=["grids_ptr"],
|
||||
)
|
||||
def _vit_mrope_setup_kernel(
|
||||
inv_freq_t_ptr,
|
||||
inv_freq_h_ptr,
|
||||
inv_freq_w_ptr,
|
||||
cos_out_ptr,
|
||||
sin_out_ptr,
|
||||
cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid
|
||||
grids_ptr, # (G, 3) int32: t, h, w per grid
|
||||
num_tokens,
|
||||
num_grids,
|
||||
half_t: tl.constexpr,
|
||||
half_h: tl.constexpr,
|
||||
half_w: tl.constexpr,
|
||||
spatial_merge: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one
|
||||
launch: compute each token's (t, h, w) position arithmetically and the
|
||||
rotation in fp32 from the per-axis inv_freq. Only each axis's mrope
|
||||
section columns are written; the rest is left uninitialized (the
|
||||
consumer's per-axis masked loads read only the section columns)."""
|
||||
pid = tl.program_id(0)
|
||||
tok = pid * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
tok_mask = tok < num_tokens
|
||||
|
||||
# Grid (segment) index per token; cu_seqlens is nondecreasing.
|
||||
seg = tl.zeros([BLOCK_N], dtype=tl.int32)
|
||||
for g in range(num_grids):
|
||||
end = tl.load(cu_seqlens_ptr + g + 1)
|
||||
seg += (tok >= end).to(tl.int32)
|
||||
|
||||
seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0)
|
||||
grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1)
|
||||
grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1)
|
||||
|
||||
local = tok - seg_start
|
||||
rem = local % (grid_h * grid_w)
|
||||
tpos = local // (grid_h * grid_w)
|
||||
# Within a frame tokens are ordered by spatial-merge block (bh, bw, mh,
|
||||
# mw), matching the reference ViT position-id permute.
|
||||
bh = rem // (spatial_merge * grid_w)
|
||||
r = rem % (spatial_merge * grid_w)
|
||||
bw = r // (spatial_merge * spatial_merge)
|
||||
r2 = r % (spatial_merge * spatial_merge)
|
||||
hpos = bh * spatial_merge + r2 // spatial_merge
|
||||
wpos = bw * spatial_merge + r2 % spatial_merge
|
||||
|
||||
d = tl.arange(0, BLOCK_D)
|
||||
half_rot = half_t + half_h + half_w
|
||||
row = (tok * half_rot)[:, None] + d[None, :]
|
||||
plane = num_tokens * half_rot
|
||||
|
||||
# Per-axis compute+store to cap live registers; the fp32 trig sequence
|
||||
# wants few elements per thread (hence small BLOCK_N, many warps).
|
||||
in_t = d < half_t
|
||||
inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0)
|
||||
freq = tpos.to(tl.float32)[:, None] * inv_t[None, :]
|
||||
mask = tok_mask[:, None] & in_t[None, :]
|
||||
tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask)
|
||||
|
||||
in_h = (d >= half_t) & (d < half_t + half_h)
|
||||
inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0)
|
||||
freq = hpos.to(tl.float32)[:, None] * inv_h[None, :]
|
||||
mask = tok_mask[:, None] & in_h[None, :]
|
||||
tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask)
|
||||
|
||||
in_w = (d >= half_t + half_h) & (d < half_rot)
|
||||
inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0)
|
||||
freq = wpos.to(tl.float32)[:, None] * inv_w[None, :]
|
||||
mask = tok_mask[:, None] & in_w[None, :]
|
||||
tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask)
|
||||
|
||||
|
||||
def vit_mrope_setup(
|
||||
inv_freq_t: torch.Tensor,
|
||||
inv_freq_h: torch.Tensor,
|
||||
inv_freq_w: torch.Tensor,
|
||||
grid_thw: list[list[int]],
|
||||
spatial_merge_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope.
|
||||
|
||||
One fused kernel: positions are computed arithmetically from the grids
|
||||
and the rotation comes from the per-axis inv_freq, so there is no
|
||||
cos/sin cache to size or grow.
|
||||
|
||||
Args:
|
||||
inv_freq_t: t-axis inverse frequencies, (t_dim // 2,).
|
||||
inv_freq_h: h-axis inverse frequencies, (h_dim // 2,).
|
||||
inv_freq_w: w-axis inverse frequencies, (w_dim // 2,).
|
||||
grid_thw: per-grid [t, h, w] patch counts (host metadata).
|
||||
spatial_merge_size: spatial merge factor of the ViT.
|
||||
|
||||
"""
|
||||
device = inv_freq_t.device
|
||||
cu = [0]
|
||||
for t, h, w in grid_thw:
|
||||
cu.append(cu[-1] + t * h * w)
|
||||
n = cu[-1]
|
||||
g = len(grid_thw)
|
||||
# grid_thw is per-request host metadata, so one H2D is unavoidable;
|
||||
# pack cu_seqlens and grids into a single pinned async copy.
|
||||
buf = async_tensor_h2d(
|
||||
cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32
|
||||
)
|
||||
cu_seqlens = buf[: g + 1]
|
||||
grids = buf[g + 1 :].view(g, 3)
|
||||
|
||||
half_t = inv_freq_t.numel()
|
||||
half_h = inv_freq_h.numel()
|
||||
half_w = inv_freq_w.numel()
|
||||
half_rot = half_t + half_h + half_w
|
||||
cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
|
||||
sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
|
||||
_vit_mrope_setup_kernel[(triton.cdiv(n, 32),)](
|
||||
inv_freq_t,
|
||||
inv_freq_h,
|
||||
inv_freq_w,
|
||||
cos,
|
||||
sin,
|
||||
cu_seqlens,
|
||||
grids,
|
||||
n,
|
||||
g,
|
||||
half_t=half_t,
|
||||
half_h=half_h,
|
||||
half_w=half_w,
|
||||
spatial_merge=spatial_merge_size,
|
||||
BLOCK_N=32,
|
||||
BLOCK_D=triton.next_power_of_2(half_rot),
|
||||
num_warps=8,
|
||||
)
|
||||
return cos, sin
|
||||
@@ -21,20 +21,16 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope_vit_setup import (
|
||||
vit_mrope_setup,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.vision import (
|
||||
get_vit_attn_backend,
|
||||
is_vit_use_data_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# ROCm caps a kernel-launch gridDim.y at 65536. The HIP flash-attn Triton
|
||||
# rotary kernel launches grid.y = cdiv(seqlen, BLOCK_M), so it fails with
|
||||
# hipErrorInvalidValue once cdiv(seqlen, BLOCK_M) > 65536. Used below to decide
|
||||
# when RoPE must be applied per video segment instead of in one launch.
|
||||
_HIP_MAX_GRID_DIM_Y = 65536
|
||||
|
||||
|
||||
class MiniMaxVLPatchEmbed(nn.Module):
|
||||
@@ -125,56 +121,13 @@ class MiniMaxVLAttention(nn.Module):
|
||||
head_size=self.head_dim,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
# ApplyRotaryEmb handles the internal cos/sin repeat and partial
|
||||
# rotation (ro_dim = half_rot_dim * 2 < head_dim for MiniMax).
|
||||
# enable_fp32_compute=True runs the rotation in fp32 (q/k upcast,
|
||||
# fp32 cos/sin), matching the reference ``_minimax_rope_applier``.
|
||||
self.apply_rotary_emb = ApplyRotaryEmb(
|
||||
enforce_enable=True, enable_fp32_compute=True
|
||||
)
|
||||
|
||||
def _apply_rotary_emb(
|
||||
self,
|
||||
qk_reshaped: torch.Tensor,
|
||||
rotary_cos: torch.Tensor,
|
||||
rotary_sin: torch.Tensor,
|
||||
seq_len: int,
|
||||
rotary_segment_lengths: list[int] | None,
|
||||
) -> torch.Tensor:
|
||||
# Default fast path (all NVIDIA inputs, and ROCm short clips/images):
|
||||
# a single rotary kernel launch. ``rotary_segment_lengths`` is only
|
||||
# populated on ROCm (see ``MiniMaxVLVisionTransformer.forward``), so
|
||||
# the per-segment path below is ROCm-only and never touches the
|
||||
# NVIDIA/CUDA code path.
|
||||
if not current_platform.is_rocm() or rotary_segment_lengths is None:
|
||||
return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin)
|
||||
|
||||
# ROCm only: the HIP flash-attn Triton rotary kernel fails with
|
||||
# hipErrorInvalidValue once grid.y = cdiv(seqlen, BLOCK_M) exceeds
|
||||
# _HIP_MAX_GRID_DIM_Y (65536). BLOCK_M is 8 for rotary_dim <= 128
|
||||
# (MiniMax-M3 vision: rotary_dim=78), giving a hard limit of
|
||||
# 65536 * BLOCK_M tokens — measured exactly as 524288 OK / 524289 fail.
|
||||
# Only long videos cross it; since vision_segment_max_frames caps each
|
||||
# segment at a few frames (<< limit), applying RoPE per segment keeps
|
||||
# every sub-call in range. Splitting on segment boundaries is
|
||||
# mathematically exact because rotary_cos/sin are precomputed per token.
|
||||
# Images and short clips stay on the single-kernel fast path above.
|
||||
rotary_dim = rotary_cos.shape[-1] * 2
|
||||
block_m = 8 if rotary_dim <= 128 else 4
|
||||
hip_rotary_max_seqlen = _HIP_MAX_GRID_DIM_Y * block_m
|
||||
if seq_len <= hip_rotary_max_seqlen or len(rotary_segment_lengths) <= 1:
|
||||
return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin)
|
||||
|
||||
qk_segments = qk_reshaped.split(rotary_segment_lengths, dim=1)
|
||||
cos_segments = rotary_cos.split(rotary_segment_lengths, dim=0)
|
||||
sin_segments = rotary_sin.split(rotary_segment_lengths, dim=0)
|
||||
return torch.cat(
|
||||
[
|
||||
self.apply_rotary_emb(qk_s, cos_s, sin_s)
|
||||
for qk_s, cos_s, sin_s in zip(qk_segments, cos_segments, sin_segments)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
# Partial 3D RoPE geometry: rot_dim is split evenly across t/h/w
|
||||
# (same formula as MiniMaxVLVisionTransformer; rot_dim may be <
|
||||
# head_dim). mrope_section is in half-dim units for triton_mrope.
|
||||
rope_dims = 2 * (self.head_dim // 2)
|
||||
axis_dim = 2 * ((rope_dims // 3) // 2)
|
||||
self.rotary_dim = 3 * axis_dim
|
||||
self.mrope_section = [axis_dim // 2] * 3
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -183,33 +136,34 @@ class MiniMaxVLAttention(nn.Module):
|
||||
rotary_cos: torch.Tensor,
|
||||
rotary_sin: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
rotary_segment_lengths: list[int] | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# x: (N, 1, embed_dim) [seq=N, batch=1, chan=embed_dim]
|
||||
x_qkv, _ = self.qkv_proj(x) # (N, 1, 3 * heads_per_part * head_dim)
|
||||
seq_len, batch_size, _ = x_qkv.shape
|
||||
|
||||
# Rearrange to (b=1, N, 3, heads, head_dim) — same as Qwen2_5_VisionAttention
|
||||
qkv = rearrange(
|
||||
x_qkv,
|
||||
"s b (three head d) -> b s three head d",
|
||||
three=3,
|
||||
head=self.num_heads_per_partition,
|
||||
)
|
||||
qk, v = qkv[:, :, :2], qkv[:, :, 2] # (b,N,2,h,d) and (b,N,h,d)
|
||||
qkv = x_qkv.view(seq_len, 3, self.num_heads_per_partition, self.head_dim)
|
||||
v = qkv[:, 2].unsqueeze(0) # (b=1, N, heads, head_dim) strided view
|
||||
|
||||
# Stack q/k → (2*b, N, heads, head_dim) for joint RoPE application.
|
||||
# rotary_cos/sin: (N, half_rot_dim) — ApplyRotaryEmb expands internally
|
||||
# and rotates only the first 2*half_rot_dim dims, passing the rest through.
|
||||
qk_reshaped = rearrange(qk, "b s two h d -> (two b) s h d", two=2).contiguous()
|
||||
qk_rotated = self._apply_rotary_emb(
|
||||
qk_reshaped, rotary_cos, rotary_sin, seq_len, rotary_segment_lengths
|
||||
# In-place partial 3D RoPE on strided q/k views of the packed qkv
|
||||
# tensor: one kernel launch, no repack/upcast copies. fp32 cos/sin
|
||||
# make the kernel compute in fp32, matching the reference
|
||||
# ``_minimax_rope_applier`` precision.
|
||||
q = qkv[:, 0].reshape(seq_len, -1)
|
||||
k = qkv[:, 1].reshape(seq_len, -1)
|
||||
triton_mrope(
|
||||
q,
|
||||
k,
|
||||
rotary_cos,
|
||||
rotary_sin,
|
||||
self.mrope_section,
|
||||
self.head_dim,
|
||||
self.rotary_dim,
|
||||
mrope_interleaved=False,
|
||||
is_neox_style=True,
|
||||
)
|
||||
qk_rotated = qk_rotated.view(
|
||||
2, batch_size, seq_len, self.num_heads_per_partition, self.head_dim
|
||||
)
|
||||
q, k = qk_rotated.unbind(dim=0) # each (b=1, N, heads, head_dim)
|
||||
q = q.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim)
|
||||
k = k.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim)
|
||||
|
||||
# Flash attention → (b, N, heads, head_dim)
|
||||
context = self.attn(
|
||||
@@ -272,7 +226,6 @@ class MiniMaxVLEncoderLayer(nn.Module):
|
||||
rotary_cos: torch.Tensor,
|
||||
rotary_sin: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
rotary_segment_lengths: list[int] | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# x: (N, 1, hidden_size)
|
||||
@@ -282,7 +235,6 @@ class MiniMaxVLEncoderLayer(nn.Module):
|
||||
rotary_cos,
|
||||
rotary_sin,
|
||||
max_seqlen,
|
||||
rotary_segment_lengths,
|
||||
sequence_lengths,
|
||||
)
|
||||
residual = x
|
||||
@@ -324,7 +276,6 @@ class MiniMaxVLEncoder(nn.Module):
|
||||
rotary_cos: torch.Tensor,
|
||||
rotary_sin: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
rotary_segment_lengths: list[int] | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
for layer in self.layers:
|
||||
@@ -334,7 +285,6 @@ class MiniMaxVLEncoder(nn.Module):
|
||||
rotary_cos,
|
||||
rotary_sin,
|
||||
max_seqlen,
|
||||
rotary_segment_lengths,
|
||||
sequence_lengths,
|
||||
)
|
||||
return x
|
||||
@@ -427,82 +377,6 @@ class MiniMaxVLVisionTransformer(nn.Module):
|
||||
# out_hidden_size needed by run_dp_sharded_mrope_vision_model
|
||||
self.out_hidden_size = embed_dim
|
||||
|
||||
# ── RoPE helpers ─────────────────────────────────────────────────────
|
||||
|
||||
def _get_3d_rope_embed(
|
||||
self, grid_t: int, grid_h: int, grid_w: int, spatial_merge_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Compute 3D RoPE frequencies for a single (T, H, W) grid.
|
||||
|
||||
Returns (T*H*W, half_rot_dim) on the same device as inv_freq buffers.
|
||||
Mirrors the reference ``_get_3d_rope_embed`` exactly.
|
||||
"""
|
||||
tokens_per_frame = grid_h * grid_w
|
||||
|
||||
tpos_ids = (
|
||||
torch.arange(grid_t, device=self.inv_freq_t.device)
|
||||
.unsqueeze(1)
|
||||
.expand(-1, tokens_per_frame)
|
||||
.flatten()
|
||||
)
|
||||
|
||||
hpos_ids = (
|
||||
torch.arange(grid_h, device=self.inv_freq_h.device)
|
||||
.unsqueeze(1)
|
||||
.expand(-1, grid_w)
|
||||
.reshape(
|
||||
grid_h // spatial_merge_size,
|
||||
spatial_merge_size,
|
||||
grid_w // spatial_merge_size,
|
||||
spatial_merge_size,
|
||||
)
|
||||
.permute(0, 2, 1, 3)
|
||||
.unsqueeze(0)
|
||||
.expand(grid_t, -1, -1, -1, -1)
|
||||
.flatten()
|
||||
)
|
||||
wpos_ids = (
|
||||
torch.arange(grid_w, device=self.inv_freq_w.device)
|
||||
.unsqueeze(0)
|
||||
.expand(grid_h, -1)
|
||||
.reshape(
|
||||
grid_h // spatial_merge_size,
|
||||
spatial_merge_size,
|
||||
grid_w // spatial_merge_size,
|
||||
spatial_merge_size,
|
||||
)
|
||||
.permute(0, 2, 1, 3)
|
||||
.unsqueeze(0)
|
||||
.expand(grid_t, -1, -1, -1, -1)
|
||||
.flatten()
|
||||
)
|
||||
|
||||
max_t = max(grid_t, 1)
|
||||
max_hw = max(grid_h, grid_w)
|
||||
|
||||
seq_t = torch.arange(
|
||||
max_t, device=self.inv_freq_t.device, dtype=self.inv_freq_t.dtype
|
||||
)
|
||||
seq_hw = torch.arange(
|
||||
max_hw, device=self.inv_freq_h.device, dtype=self.inv_freq_h.dtype
|
||||
)
|
||||
|
||||
freqs_t = torch.outer(seq_t, self.inv_freq_t) # (max_t, t_dim/2)
|
||||
freqs_h = torch.outer(seq_hw, self.inv_freq_h) # (max_hw, h_dim/2)
|
||||
freqs_w = torch.outer(seq_hw, self.inv_freq_w) # (max_hw, w_dim/2)
|
||||
|
||||
return torch.cat(
|
||||
[freqs_t[tpos_ids], freqs_h[hpos_ids], freqs_w[wpos_ids]], dim=-1
|
||||
) # (T*H*W, half_rot_dim)
|
||||
|
||||
def _get_rope_embed_3d(
|
||||
self, grid_thw: list[list[int]], spatial_merge_size: int
|
||||
) -> torch.Tensor:
|
||||
embeds = [
|
||||
self._get_3d_rope_embed(t, h, w, spatial_merge_size) for t, h, w in grid_thw
|
||||
]
|
||||
return torch.cat(embeds, dim=0) # (total_N, half_rot_dim)
|
||||
|
||||
# ── Frame-limit helper (mirrors the reference) ───────────────────────
|
||||
|
||||
def _apply_max_frames_limit(self, grid_thw: list[list[int]]) -> list[list[int]]:
|
||||
@@ -557,21 +431,19 @@ class MiniMaxVLVisionTransformer(nn.Module):
|
||||
hidden.device,
|
||||
)
|
||||
|
||||
# 3D RoPE: (total_N, half_rot_dim); ApplyRotaryEmb expands internally
|
||||
freqs = self._get_rope_embed_3d(limited, self.spatial_merge_size)
|
||||
freqs = freqs.to(device=hidden.device)
|
||||
# Keep cos/sin in fp32; ApplyRotaryEmb(enable_fp32_compute=True) runs the
|
||||
# rotation in fp32 to match the reference precision.
|
||||
rotary_cos, rotary_sin = freqs.cos(), freqs.sin()
|
||||
# 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for
|
||||
# triton_mrope from one fused setup kernel. Kept fp32 so the kernel
|
||||
# computes the rotation in fp32, matching the reference.
|
||||
rotary_cos, rotary_sin = vit_mrope_setup(
|
||||
self.inv_freq_t,
|
||||
self.inv_freq_h,
|
||||
self.inv_freq_w,
|
||||
limited,
|
||||
self.spatial_merge_size,
|
||||
)
|
||||
|
||||
# Encoder expects (N, 1, hidden_size) — add batch dim
|
||||
hidden = hidden.unsqueeze(1)
|
||||
# On ROCm, the flash_attn Triton rotary kernel can fail with
|
||||
# hipErrorInvalidValue when seqlen is very large, e.g. 192k video
|
||||
# tokens; pass per-segment lengths so RoPE can be applied in chunks.
|
||||
# On other platforms leave it None -> single-kernel fast path, so the
|
||||
# NVIDIA/CUDA code path is unchanged.
|
||||
rotary_segment_lengths = lens if current_platform.is_rocm() else None
|
||||
|
||||
hidden = self.encoder(
|
||||
hidden,
|
||||
@@ -579,7 +451,6 @@ class MiniMaxVLVisionTransformer(nn.Module):
|
||||
rotary_cos,
|
||||
rotary_sin,
|
||||
max_seqlen,
|
||||
rotary_segment_lengths,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
hidden = hidden.squeeze(1) # back to (total_N, hidden_size)
|
||||
|
||||
Reference in New Issue
Block a user