[Minimax-M3][Perf] Use triton_mrope for vision tower + int64 offset fix for triton_mrope (#58526)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Kimi <noreply@moonshot.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Kimi
Isotr0py
parent
ade1056ee5
commit
16070ed7e9
@@ -20,6 +20,8 @@ def _triton_mrope_forward(
|
||||
cos,
|
||||
sin,
|
||||
num_tokens,
|
||||
q_token_stride,
|
||||
k_token_stride,
|
||||
n_qh: tl.constexpr,
|
||||
n_kh: tl.constexpr,
|
||||
hd: tl.constexpr,
|
||||
@@ -39,9 +41,10 @@ def _triton_mrope_forward(
|
||||
# and supports cos and sin cache with shape (3, num_tokens, rotary_dim // 2)
|
||||
# instead of (3, bsz, seq_len, head_dim), also supports interleaved rotary
|
||||
pid = tl.program_id(0)
|
||||
# locate start address
|
||||
q_ptr = q_ptr + pid * (n_qh * hd)
|
||||
k_ptr = k_ptr + pid * (n_kh * hd)
|
||||
# locate start address (int64: num_tokens * row_stride exceeds 2^31 for
|
||||
# worst-case ViT forwards, e.g. MiniMax M3 packs ~768k patches/item)
|
||||
q_ptr = q_ptr + pid.to(tl.int64) * q_token_stride
|
||||
k_ptr = k_ptr + pid.to(tl.int64) * k_token_stride
|
||||
|
||||
# ####################################################################
|
||||
# get the cos(mθ_{i...d/2}) and sin(mθ_{i...d/2}) for token position
|
||||
@@ -186,6 +189,10 @@ def triton_mrope(
|
||||
adjacent (GPT-J) layout.
|
||||
|
||||
"""
|
||||
# The kernel indexes each token row through an explicit row stride and
|
||||
# requires dense rows.
|
||||
assert q.dim() == 2 and q.stride(-1) == 1
|
||||
assert k.dim() == 2 and k.stride(-1) == 1
|
||||
n_row, n_q_head_head_dim = q.shape
|
||||
n_q_head = n_q_head_head_dim // head_size
|
||||
n_kv_head = k.shape[1] // head_size
|
||||
@@ -193,10 +200,6 @@ def triton_mrope(
|
||||
pad_n_q_head = triton.next_power_of_2(n_q_head)
|
||||
pad_n_kv_head = triton.next_power_of_2(n_kv_head)
|
||||
|
||||
# ensure tensors passed into the kernel are contiguous.
|
||||
# It will be no-op if they are already contiguous
|
||||
q = q.contiguous()
|
||||
k = k.contiguous()
|
||||
cos = cos.contiguous()
|
||||
sin = sin.contiguous()
|
||||
|
||||
@@ -211,6 +214,8 @@ def triton_mrope(
|
||||
cos,
|
||||
sin,
|
||||
n_row,
|
||||
q.stride(0),
|
||||
k.stride(0),
|
||||
n_q_head,
|
||||
n_kv_head,
|
||||
head_size,
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr
|
||||
# is an offset slice of the packed H2D buffer, so its 16B alignment varies
|
||||
# with grid count; without these the kernel recompiles mid-serving.
|
||||
@triton.jit(
|
||||
do_not_specialize=["num_tokens", "num_grids"],
|
||||
do_not_specialize_on_alignment=["grids_ptr"],
|
||||
)
|
||||
def _vit_mrope_setup_kernel(
|
||||
inv_freq_t_ptr,
|
||||
inv_freq_h_ptr,
|
||||
inv_freq_w_ptr,
|
||||
cos_out_ptr,
|
||||
sin_out_ptr,
|
||||
cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid
|
||||
grids_ptr, # (G, 3) int32: t, h, w per grid
|
||||
num_tokens,
|
||||
num_grids,
|
||||
half_t: tl.constexpr,
|
||||
half_h: tl.constexpr,
|
||||
half_w: tl.constexpr,
|
||||
spatial_merge: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one
|
||||
launch: compute each token's (t, h, w) position arithmetically and the
|
||||
rotation in fp32 from the per-axis inv_freq. Only each axis's mrope
|
||||
section columns are written; the rest is left uninitialized (the
|
||||
consumer's per-axis masked loads read only the section columns)."""
|
||||
pid = tl.program_id(0)
|
||||
tok = pid * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
tok_mask = tok < num_tokens
|
||||
|
||||
# Grid (segment) index per token; cu_seqlens is nondecreasing.
|
||||
seg = tl.zeros([BLOCK_N], dtype=tl.int32)
|
||||
for g in range(num_grids):
|
||||
end = tl.load(cu_seqlens_ptr + g + 1)
|
||||
seg += (tok >= end).to(tl.int32)
|
||||
|
||||
seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0)
|
||||
grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1)
|
||||
grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1)
|
||||
|
||||
local = tok - seg_start
|
||||
rem = local % (grid_h * grid_w)
|
||||
tpos = local // (grid_h * grid_w)
|
||||
# Within a frame tokens are ordered by spatial-merge block (bh, bw, mh,
|
||||
# mw), matching the reference ViT position-id permute.
|
||||
bh = rem // (spatial_merge * grid_w)
|
||||
r = rem % (spatial_merge * grid_w)
|
||||
bw = r // (spatial_merge * spatial_merge)
|
||||
r2 = r % (spatial_merge * spatial_merge)
|
||||
hpos = bh * spatial_merge + r2 // spatial_merge
|
||||
wpos = bw * spatial_merge + r2 % spatial_merge
|
||||
|
||||
d = tl.arange(0, BLOCK_D)
|
||||
half_rot = half_t + half_h + half_w
|
||||
row = (tok * half_rot)[:, None] + d[None, :]
|
||||
plane = num_tokens * half_rot
|
||||
|
||||
# Per-axis compute+store to cap live registers; the fp32 trig sequence
|
||||
# wants few elements per thread (hence small BLOCK_N, many warps).
|
||||
in_t = d < half_t
|
||||
inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0)
|
||||
freq = tpos.to(tl.float32)[:, None] * inv_t[None, :]
|
||||
mask = tok_mask[:, None] & in_t[None, :]
|
||||
tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask)
|
||||
|
||||
in_h = (d >= half_t) & (d < half_t + half_h)
|
||||
inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0)
|
||||
freq = hpos.to(tl.float32)[:, None] * inv_h[None, :]
|
||||
mask = tok_mask[:, None] & in_h[None, :]
|
||||
tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask)
|
||||
|
||||
in_w = (d >= half_t + half_h) & (d < half_rot)
|
||||
inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0)
|
||||
freq = wpos.to(tl.float32)[:, None] * inv_w[None, :]
|
||||
mask = tok_mask[:, None] & in_w[None, :]
|
||||
tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask)
|
||||
tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask)
|
||||
|
||||
|
||||
def vit_mrope_setup(
|
||||
inv_freq_t: torch.Tensor,
|
||||
inv_freq_h: torch.Tensor,
|
||||
inv_freq_w: torch.Tensor,
|
||||
grid_thw: list[list[int]],
|
||||
spatial_merge_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope.
|
||||
|
||||
One fused kernel: positions are computed arithmetically from the grids
|
||||
and the rotation comes from the per-axis inv_freq, so there is no
|
||||
cos/sin cache to size or grow.
|
||||
|
||||
Args:
|
||||
inv_freq_t: t-axis inverse frequencies, (t_dim // 2,).
|
||||
inv_freq_h: h-axis inverse frequencies, (h_dim // 2,).
|
||||
inv_freq_w: w-axis inverse frequencies, (w_dim // 2,).
|
||||
grid_thw: per-grid [t, h, w] patch counts (host metadata).
|
||||
spatial_merge_size: spatial merge factor of the ViT.
|
||||
|
||||
"""
|
||||
device = inv_freq_t.device
|
||||
cu = [0]
|
||||
for t, h, w in grid_thw:
|
||||
cu.append(cu[-1] + t * h * w)
|
||||
n = cu[-1]
|
||||
g = len(grid_thw)
|
||||
# grid_thw is per-request host metadata, so one H2D is unavoidable;
|
||||
# pack cu_seqlens and grids into a single pinned async copy.
|
||||
buf = async_tensor_h2d(
|
||||
cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32
|
||||
)
|
||||
cu_seqlens = buf[: g + 1]
|
||||
grids = buf[g + 1 :].view(g, 3)
|
||||
|
||||
half_t = inv_freq_t.numel()
|
||||
half_h = inv_freq_h.numel()
|
||||
half_w = inv_freq_w.numel()
|
||||
half_rot = half_t + half_h + half_w
|
||||
cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
|
||||
sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32)
|
||||
_vit_mrope_setup_kernel[(triton.cdiv(n, 32),)](
|
||||
inv_freq_t,
|
||||
inv_freq_h,
|
||||
inv_freq_w,
|
||||
cos,
|
||||
sin,
|
||||
cu_seqlens,
|
||||
grids,
|
||||
n,
|
||||
g,
|
||||
half_t=half_t,
|
||||
half_h=half_h,
|
||||
half_w=half_w,
|
||||
spatial_merge=spatial_merge_size,
|
||||
BLOCK_N=32,
|
||||
BLOCK_D=triton.next_power_of_2(half_rot),
|
||||
num_warps=8,
|
||||
)
|
||||
return cos, sin
|
||||
Reference in New Issue
Block a user