diff --git a/tests/kernels/core/test_mrope.py b/tests/kernels/core/test_mrope.py index 0e355d7cd3..e590d8264c 100644 --- a/tests/kernels/core/test_mrope.py +++ b/tests/kernels/core/test_mrope.py @@ -6,7 +6,10 @@ import pytest import torch from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.rotary_embedding.mrope import apply_interleaved_rope +from vllm.model_executor.layers.rotary_embedding.mrope import ( + apply_interleaved_rope, + triton_mrope, +) from vllm.platforms import current_platform from vllm.transformers_utils.config import get_config from vllm.utils.torch_utils import set_random_seed @@ -174,6 +177,75 @@ def test_mrope( torch.testing.assert_close(query_native, query_cuda, atol=atol, rtol=rtol) torch.testing.assert_close(key_native, key_cuda, atol=atol, rtol=rtol) + # Strided q/k views of a packed buffer (the MiniMax M3 ViT usage): + # bitwise-identical to the contiguous run, neighboring columns untouched. + pad = 16 + packed = torch.zeros( + num_tokens, + query.shape[1] + key.shape[1] + pad, + dtype=dtype, + device=device, + ) + packed[:, : query.shape[1]] = query + packed[:, query.shape[1] : query.shape[1] + key.shape[1]] = key + sentinel = packed[:, -pad:].clone() + q_strided = packed[:, : query.shape[1]] + k_strided = packed[:, query.shape[1] : query.shape[1] + key.shape[1]] + mrope_helper_class.forward_cuda(positions, q_strided, k_strided) + torch.testing.assert_close(q_strided, query_cuda, atol=0, rtol=0) + torch.testing.assert_close(k_strided, key_cuda, atol=0, rtol=0) + torch.testing.assert_close(packed[:, -pad:], sentinel, atol=0, rtol=0) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests." +) +def test_mrope_row_offset_int64(): + # MiniMax M3 ViT worst case: a max-size video item packs ~768k patches, + # and num_tokens * packed-qkv row stride (3840) exceeds 2^31 at 559,241 + # rows. Row-base math must be int64 or those rows wrap to negative + # offsets (illegal memory access at startup profiling). + set_random_seed(0) + n_qh, n_kh, hd, rd = 16, 16, 80, 78 + mrope_section = [13, 13, 13] + row_stride = (n_qh + n_kh + n_kh) * hd # packed qkv row + num_tokens = 2**31 // row_stride + 8 + free, _ = current_platform.mem_get_info() + if free < 2 * num_tokens * row_stride * 2: + pytest.skip("not enough free device memory") + + packed = torch.empty(num_tokens, row_stride, dtype=torch.bfloat16, device=device) + q_fill = torch.randn(n_qh * hd, dtype=torch.bfloat16, device=device) + k_fill = torch.randn(n_kh * hd, dtype=torch.bfloat16, device=device) + packed[:, : n_qh * hd] = q_fill + packed[:, n_qh * hd : (n_qh + n_kh) * hd] = k_fill + half_rd = rd // 2 + cos = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device) + sin = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device) + q = packed[:, : n_qh * hd] + k = packed[:, n_qh * hd : (n_qh + n_kh) * hd] + triton_mrope(q, k, cos, sin, mrope_section, hd, rd, False, True) + + t_end = mrope_section[0] + h_end = t_end + mrope_section[1] + for r in (0, num_tokens - 8, num_tokens - 1): + cos_row = torch.cat( + [cos[0, r, :t_end], cos[1, r, t_end:h_end], cos[2, r, h_end:]] + ) + sin_row = torch.cat( + [sin[0, r, :t_end], sin[1, r, t_end:h_end], sin[2, r, h_end:]] + ) + for fill, out in ((q_fill, q[r]), (k_fill, k[r])): + row = fill.view(-1, hd).float() + r1, r2 = row[:, :half_rd], row[:, half_rd:rd] + exp = torch.cat( + [r1 * cos_row - r2 * sin_row, r2 * cos_row + r1 * sin_row, row[:, rd:]], + dim=1, + ) + torch.testing.assert_close( + out, exp.view(-1).to(torch.bfloat16), atol=1e-2, rtol=1.6e-2 + ) + @pytest.mark.skipif( not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests." diff --git a/vllm/model_executor/layers/rotary_embedding/mrope.py b/vllm/model_executor/layers/rotary_embedding/mrope.py index 42013926d7..11882856e0 100644 --- a/vllm/model_executor/layers/rotary_embedding/mrope.py +++ b/vllm/model_executor/layers/rotary_embedding/mrope.py @@ -20,6 +20,8 @@ def _triton_mrope_forward( cos, sin, num_tokens, + q_token_stride, + k_token_stride, n_qh: tl.constexpr, n_kh: tl.constexpr, hd: tl.constexpr, @@ -39,9 +41,10 @@ def _triton_mrope_forward( # and supports cos and sin cache with shape (3, num_tokens, rotary_dim // 2) # instead of (3, bsz, seq_len, head_dim), also supports interleaved rotary pid = tl.program_id(0) - # locate start address - q_ptr = q_ptr + pid * (n_qh * hd) - k_ptr = k_ptr + pid * (n_kh * hd) + # locate start address (int64: num_tokens * row_stride exceeds 2^31 for + # worst-case ViT forwards, e.g. MiniMax M3 packs ~768k patches/item) + q_ptr = q_ptr + pid.to(tl.int64) * q_token_stride + k_ptr = k_ptr + pid.to(tl.int64) * k_token_stride # #################################################################### # get the cos(mθ_{i...d/2}) and sin(mθ_{i...d/2}) for token position @@ -186,6 +189,10 @@ def triton_mrope( adjacent (GPT-J) layout. """ + # The kernel indexes each token row through an explicit row stride and + # requires dense rows. + assert q.dim() == 2 and q.stride(-1) == 1 + assert k.dim() == 2 and k.stride(-1) == 1 n_row, n_q_head_head_dim = q.shape n_q_head = n_q_head_head_dim // head_size n_kv_head = k.shape[1] // head_size @@ -193,10 +200,6 @@ def triton_mrope( pad_n_q_head = triton.next_power_of_2(n_q_head) pad_n_kv_head = triton.next_power_of_2(n_kv_head) - # ensure tensors passed into the kernel are contiguous. - # It will be no-op if they are already contiguous - q = q.contiguous() - k = k.contiguous() cos = cos.contiguous() sin = sin.contiguous() @@ -211,6 +214,8 @@ def triton_mrope( cos, sin, n_row, + q.stride(0), + k.stride(0), n_q_head, n_kv_head, head_size, diff --git a/vllm/model_executor/layers/rotary_embedding/mrope_vit_setup.py b/vllm/model_executor/layers/rotary_embedding/mrope_vit_setup.py new file mode 100644 index 0000000000..6453e8c835 --- /dev/null +++ b/vllm/model_executor/layers/rotary_embedding/mrope_vit_setup.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import async_tensor_h2d + + +# num_tokens/num_grids vary per request (incl. the ==1 class) and grids_ptr +# is an offset slice of the packed H2D buffer, so its 16B alignment varies +# with grid count; without these the kernel recompiles mid-serving. +@triton.jit( + do_not_specialize=["num_tokens", "num_grids"], + do_not_specialize_on_alignment=["grids_ptr"], +) +def _vit_mrope_setup_kernel( + inv_freq_t_ptr, + inv_freq_h_ptr, + inv_freq_w_ptr, + cos_out_ptr, + sin_out_ptr, + cu_seqlens_ptr, # (G + 1,) int32 token boundaries per grid + grids_ptr, # (G, 3) int32: t, h, w per grid + num_tokens, + num_grids, + half_t: tl.constexpr, + half_h: tl.constexpr, + half_w: tl.constexpr, + spatial_merge: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Build triton_mrope's (3, N, half_rot) cos/sin t/h/w planes in one + launch: compute each token's (t, h, w) position arithmetically and the + rotation in fp32 from the per-axis inv_freq. Only each axis's mrope + section columns are written; the rest is left uninitialized (the + consumer's per-axis masked loads read only the section columns).""" + pid = tl.program_id(0) + tok = pid * BLOCK_N + tl.arange(0, BLOCK_N) + tok_mask = tok < num_tokens + + # Grid (segment) index per token; cu_seqlens is nondecreasing. + seg = tl.zeros([BLOCK_N], dtype=tl.int32) + for g in range(num_grids): + end = tl.load(cu_seqlens_ptr + g + 1) + seg += (tok >= end).to(tl.int32) + + seg_start = tl.load(cu_seqlens_ptr + seg, mask=tok_mask, other=0) + grid_h = tl.load(grids_ptr + seg * 3 + 1, mask=tok_mask, other=1) + grid_w = tl.load(grids_ptr + seg * 3 + 2, mask=tok_mask, other=1) + + local = tok - seg_start + rem = local % (grid_h * grid_w) + tpos = local // (grid_h * grid_w) + # Within a frame tokens are ordered by spatial-merge block (bh, bw, mh, + # mw), matching the reference ViT position-id permute. + bh = rem // (spatial_merge * grid_w) + r = rem % (spatial_merge * grid_w) + bw = r // (spatial_merge * spatial_merge) + r2 = r % (spatial_merge * spatial_merge) + hpos = bh * spatial_merge + r2 // spatial_merge + wpos = bw * spatial_merge + r2 % spatial_merge + + d = tl.arange(0, BLOCK_D) + half_rot = half_t + half_h + half_w + row = (tok * half_rot)[:, None] + d[None, :] + plane = num_tokens * half_rot + + # Per-axis compute+store to cap live registers; the fp32 trig sequence + # wants few elements per thread (hence small BLOCK_N, many warps). + in_t = d < half_t + inv_t = tl.load(inv_freq_t_ptr + d, mask=in_t, other=0.0) + freq = tpos.to(tl.float32)[:, None] * inv_t[None, :] + mask = tok_mask[:, None] & in_t[None, :] + tl.store(cos_out_ptr + row, tl.cos(freq), mask=mask) + tl.store(sin_out_ptr + row, tl.sin(freq), mask=mask) + + in_h = (d >= half_t) & (d < half_t + half_h) + inv_h = tl.load(inv_freq_h_ptr + d - half_t, mask=in_h, other=0.0) + freq = hpos.to(tl.float32)[:, None] * inv_h[None, :] + mask = tok_mask[:, None] & in_h[None, :] + tl.store(cos_out_ptr + plane + row, tl.cos(freq), mask=mask) + tl.store(sin_out_ptr + plane + row, tl.sin(freq), mask=mask) + + in_w = (d >= half_t + half_h) & (d < half_rot) + inv_w = tl.load(inv_freq_w_ptr + d - half_t - half_h, mask=in_w, other=0.0) + freq = wpos.to(tl.float32)[:, None] * inv_w[None, :] + mask = tok_mask[:, None] & in_w[None, :] + tl.store(cos_out_ptr + 2 * plane + row, tl.cos(freq), mask=mask) + tl.store(sin_out_ptr + 2 * plane + row, tl.sin(freq), mask=mask) + + +def vit_mrope_setup( + inv_freq_t: torch.Tensor, + inv_freq_h: torch.Tensor, + inv_freq_w: torch.Tensor, + grid_thw: list[list[int]], + spatial_merge_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """(3, total_N, half_rot_dim) fp32 cos/sin t/h/w planes for triton_mrope. + + One fused kernel: positions are computed arithmetically from the grids + and the rotation comes from the per-axis inv_freq, so there is no + cos/sin cache to size or grow. + + Args: + inv_freq_t: t-axis inverse frequencies, (t_dim // 2,). + inv_freq_h: h-axis inverse frequencies, (h_dim // 2,). + inv_freq_w: w-axis inverse frequencies, (w_dim // 2,). + grid_thw: per-grid [t, h, w] patch counts (host metadata). + spatial_merge_size: spatial merge factor of the ViT. + + """ + device = inv_freq_t.device + cu = [0] + for t, h, w in grid_thw: + cu.append(cu[-1] + t * h * w) + n = cu[-1] + g = len(grid_thw) + # grid_thw is per-request host metadata, so one H2D is unavoidable; + # pack cu_seqlens and grids into a single pinned async copy. + buf = async_tensor_h2d( + cu + [d for grid in grid_thw for d in grid], device=device, dtype=torch.int32 + ) + cu_seqlens = buf[: g + 1] + grids = buf[g + 1 :].view(g, 3) + + half_t = inv_freq_t.numel() + half_h = inv_freq_h.numel() + half_w = inv_freq_w.numel() + half_rot = half_t + half_h + half_w + cos = torch.empty(3, n, half_rot, device=device, dtype=torch.float32) + sin = torch.empty(3, n, half_rot, device=device, dtype=torch.float32) + _vit_mrope_setup_kernel[(triton.cdiv(n, 32),)]( + inv_freq_t, + inv_freq_h, + inv_freq_w, + cos, + sin, + cu_seqlens, + grids, + n, + g, + half_t=half_t, + half_h=half_h, + half_w=half_w, + spatial_merge=spatial_merge_size, + BLOCK_N=32, + BLOCK_D=triton.next_power_of_2(half_rot), + num_warps=8, + ) + return cos, sin diff --git a/vllm/models/minimax_m3/common/vision_tower.py b/vllm/models/minimax_m3/common/vision_tower.py index 468fee4237..ce56a179d0 100644 --- a/vllm/models/minimax_m3/common/vision_tower.py +++ b/vllm/models/minimax_m3/common/vision_tower.py @@ -21,20 +21,16 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb +from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope +from vllm.model_executor.layers.rotary_embedding.mrope_vit_setup import ( + vit_mrope_setup, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.utils import maybe_prefix from vllm.model_executor.models.vision import ( get_vit_attn_backend, is_vit_use_data_parallel, ) -from vllm.platforms import current_platform - -# ROCm caps a kernel-launch gridDim.y at 65536. The HIP flash-attn Triton -# rotary kernel launches grid.y = cdiv(seqlen, BLOCK_M), so it fails with -# hipErrorInvalidValue once cdiv(seqlen, BLOCK_M) > 65536. Used below to decide -# when RoPE must be applied per video segment instead of in one launch. -_HIP_MAX_GRID_DIM_Y = 65536 class MiniMaxVLPatchEmbed(nn.Module): @@ -125,56 +121,13 @@ class MiniMaxVLAttention(nn.Module): head_size=self.head_dim, prefix=f"{prefix}.attn", ) - # ApplyRotaryEmb handles the internal cos/sin repeat and partial - # rotation (ro_dim = half_rot_dim * 2 < head_dim for MiniMax). - # enable_fp32_compute=True runs the rotation in fp32 (q/k upcast, - # fp32 cos/sin), matching the reference ``_minimax_rope_applier``. - self.apply_rotary_emb = ApplyRotaryEmb( - enforce_enable=True, enable_fp32_compute=True - ) - - def _apply_rotary_emb( - self, - qk_reshaped: torch.Tensor, - rotary_cos: torch.Tensor, - rotary_sin: torch.Tensor, - seq_len: int, - rotary_segment_lengths: list[int] | None, - ) -> torch.Tensor: - # Default fast path (all NVIDIA inputs, and ROCm short clips/images): - # a single rotary kernel launch. ``rotary_segment_lengths`` is only - # populated on ROCm (see ``MiniMaxVLVisionTransformer.forward``), so - # the per-segment path below is ROCm-only and never touches the - # NVIDIA/CUDA code path. - if not current_platform.is_rocm() or rotary_segment_lengths is None: - return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin) - - # ROCm only: the HIP flash-attn Triton rotary kernel fails with - # hipErrorInvalidValue once grid.y = cdiv(seqlen, BLOCK_M) exceeds - # _HIP_MAX_GRID_DIM_Y (65536). BLOCK_M is 8 for rotary_dim <= 128 - # (MiniMax-M3 vision: rotary_dim=78), giving a hard limit of - # 65536 * BLOCK_M tokens — measured exactly as 524288 OK / 524289 fail. - # Only long videos cross it; since vision_segment_max_frames caps each - # segment at a few frames (<< limit), applying RoPE per segment keeps - # every sub-call in range. Splitting on segment boundaries is - # mathematically exact because rotary_cos/sin are precomputed per token. - # Images and short clips stay on the single-kernel fast path above. - rotary_dim = rotary_cos.shape[-1] * 2 - block_m = 8 if rotary_dim <= 128 else 4 - hip_rotary_max_seqlen = _HIP_MAX_GRID_DIM_Y * block_m - if seq_len <= hip_rotary_max_seqlen or len(rotary_segment_lengths) <= 1: - return self.apply_rotary_emb(qk_reshaped, rotary_cos, rotary_sin) - - qk_segments = qk_reshaped.split(rotary_segment_lengths, dim=1) - cos_segments = rotary_cos.split(rotary_segment_lengths, dim=0) - sin_segments = rotary_sin.split(rotary_segment_lengths, dim=0) - return torch.cat( - [ - self.apply_rotary_emb(qk_s, cos_s, sin_s) - for qk_s, cos_s, sin_s in zip(qk_segments, cos_segments, sin_segments) - ], - dim=1, - ) + # Partial 3D RoPE geometry: rot_dim is split evenly across t/h/w + # (same formula as MiniMaxVLVisionTransformer; rot_dim may be < + # head_dim). mrope_section is in half-dim units for triton_mrope. + rope_dims = 2 * (self.head_dim // 2) + axis_dim = 2 * ((rope_dims // 3) // 2) + self.rotary_dim = 3 * axis_dim + self.mrope_section = [axis_dim // 2] * 3 def forward( self, @@ -183,33 +136,34 @@ class MiniMaxVLAttention(nn.Module): rotary_cos: torch.Tensor, rotary_sin: torch.Tensor, max_seqlen: torch.Tensor, - rotary_segment_lengths: list[int] | None = None, sequence_lengths: torch.Tensor | None = None, ) -> torch.Tensor: # x: (N, 1, embed_dim) [seq=N, batch=1, chan=embed_dim] x_qkv, _ = self.qkv_proj(x) # (N, 1, 3 * heads_per_part * head_dim) seq_len, batch_size, _ = x_qkv.shape - # Rearrange to (b=1, N, 3, heads, head_dim) — same as Qwen2_5_VisionAttention - qkv = rearrange( - x_qkv, - "s b (three head d) -> b s three head d", - three=3, - head=self.num_heads_per_partition, - ) - qk, v = qkv[:, :, :2], qkv[:, :, 2] # (b,N,2,h,d) and (b,N,h,d) + qkv = x_qkv.view(seq_len, 3, self.num_heads_per_partition, self.head_dim) + v = qkv[:, 2].unsqueeze(0) # (b=1, N, heads, head_dim) strided view - # Stack q/k → (2*b, N, heads, head_dim) for joint RoPE application. - # rotary_cos/sin: (N, half_rot_dim) — ApplyRotaryEmb expands internally - # and rotates only the first 2*half_rot_dim dims, passing the rest through. - qk_reshaped = rearrange(qk, "b s two h d -> (two b) s h d", two=2).contiguous() - qk_rotated = self._apply_rotary_emb( - qk_reshaped, rotary_cos, rotary_sin, seq_len, rotary_segment_lengths + # In-place partial 3D RoPE on strided q/k views of the packed qkv + # tensor: one kernel launch, no repack/upcast copies. fp32 cos/sin + # make the kernel compute in fp32, matching the reference + # ``_minimax_rope_applier`` precision. + q = qkv[:, 0].reshape(seq_len, -1) + k = qkv[:, 1].reshape(seq_len, -1) + triton_mrope( + q, + k, + rotary_cos, + rotary_sin, + self.mrope_section, + self.head_dim, + self.rotary_dim, + mrope_interleaved=False, + is_neox_style=True, ) - qk_rotated = qk_rotated.view( - 2, batch_size, seq_len, self.num_heads_per_partition, self.head_dim - ) - q, k = qk_rotated.unbind(dim=0) # each (b=1, N, heads, head_dim) + q = q.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim) + k = k.view(batch_size, seq_len, self.num_heads_per_partition, self.head_dim) # Flash attention → (b, N, heads, head_dim) context = self.attn( @@ -272,7 +226,6 @@ class MiniMaxVLEncoderLayer(nn.Module): rotary_cos: torch.Tensor, rotary_sin: torch.Tensor, max_seqlen: torch.Tensor, - rotary_segment_lengths: list[int] | None = None, sequence_lengths: torch.Tensor | None = None, ) -> torch.Tensor: # x: (N, 1, hidden_size) @@ -282,7 +235,6 @@ class MiniMaxVLEncoderLayer(nn.Module): rotary_cos, rotary_sin, max_seqlen, - rotary_segment_lengths, sequence_lengths, ) residual = x @@ -324,7 +276,6 @@ class MiniMaxVLEncoder(nn.Module): rotary_cos: torch.Tensor, rotary_sin: torch.Tensor, max_seqlen: torch.Tensor, - rotary_segment_lengths: list[int] | None = None, sequence_lengths: torch.Tensor | None = None, ) -> torch.Tensor: for layer in self.layers: @@ -334,7 +285,6 @@ class MiniMaxVLEncoder(nn.Module): rotary_cos, rotary_sin, max_seqlen, - rotary_segment_lengths, sequence_lengths, ) return x @@ -427,82 +377,6 @@ class MiniMaxVLVisionTransformer(nn.Module): # out_hidden_size needed by run_dp_sharded_mrope_vision_model self.out_hidden_size = embed_dim - # ── RoPE helpers ───────────────────────────────────────────────────── - - def _get_3d_rope_embed( - self, grid_t: int, grid_h: int, grid_w: int, spatial_merge_size: int - ) -> torch.Tensor: - """Compute 3D RoPE frequencies for a single (T, H, W) grid. - - Returns (T*H*W, half_rot_dim) on the same device as inv_freq buffers. - Mirrors the reference ``_get_3d_rope_embed`` exactly. - """ - tokens_per_frame = grid_h * grid_w - - tpos_ids = ( - torch.arange(grid_t, device=self.inv_freq_t.device) - .unsqueeze(1) - .expand(-1, tokens_per_frame) - .flatten() - ) - - hpos_ids = ( - torch.arange(grid_h, device=self.inv_freq_h.device) - .unsqueeze(1) - .expand(-1, grid_w) - .reshape( - grid_h // spatial_merge_size, - spatial_merge_size, - grid_w // spatial_merge_size, - spatial_merge_size, - ) - .permute(0, 2, 1, 3) - .unsqueeze(0) - .expand(grid_t, -1, -1, -1, -1) - .flatten() - ) - wpos_ids = ( - torch.arange(grid_w, device=self.inv_freq_w.device) - .unsqueeze(0) - .expand(grid_h, -1) - .reshape( - grid_h // spatial_merge_size, - spatial_merge_size, - grid_w // spatial_merge_size, - spatial_merge_size, - ) - .permute(0, 2, 1, 3) - .unsqueeze(0) - .expand(grid_t, -1, -1, -1, -1) - .flatten() - ) - - max_t = max(grid_t, 1) - max_hw = max(grid_h, grid_w) - - seq_t = torch.arange( - max_t, device=self.inv_freq_t.device, dtype=self.inv_freq_t.dtype - ) - seq_hw = torch.arange( - max_hw, device=self.inv_freq_h.device, dtype=self.inv_freq_h.dtype - ) - - freqs_t = torch.outer(seq_t, self.inv_freq_t) # (max_t, t_dim/2) - freqs_h = torch.outer(seq_hw, self.inv_freq_h) # (max_hw, h_dim/2) - freqs_w = torch.outer(seq_hw, self.inv_freq_w) # (max_hw, w_dim/2) - - return torch.cat( - [freqs_t[tpos_ids], freqs_h[hpos_ids], freqs_w[wpos_ids]], dim=-1 - ) # (T*H*W, half_rot_dim) - - def _get_rope_embed_3d( - self, grid_thw: list[list[int]], spatial_merge_size: int - ) -> torch.Tensor: - embeds = [ - self._get_3d_rope_embed(t, h, w, spatial_merge_size) for t, h, w in grid_thw - ] - return torch.cat(embeds, dim=0) # (total_N, half_rot_dim) - # ── Frame-limit helper (mirrors the reference) ─────────────────────── def _apply_max_frames_limit(self, grid_thw: list[list[int]]) -> list[list[int]]: @@ -557,21 +431,19 @@ class MiniMaxVLVisionTransformer(nn.Module): hidden.device, ) - # 3D RoPE: (total_N, half_rot_dim); ApplyRotaryEmb expands internally - freqs = self._get_rope_embed_3d(limited, self.spatial_merge_size) - freqs = freqs.to(device=hidden.device) - # Keep cos/sin in fp32; ApplyRotaryEmb(enable_fp32_compute=True) runs the - # rotation in fp32 to match the reference precision. - rotary_cos, rotary_sin = freqs.cos(), freqs.sin() + # 3D RoPE cos/sin: (3, total_N, half_rot_dim) fp32 t/h/w planes for + # triton_mrope from one fused setup kernel. Kept fp32 so the kernel + # computes the rotation in fp32, matching the reference. + rotary_cos, rotary_sin = vit_mrope_setup( + self.inv_freq_t, + self.inv_freq_h, + self.inv_freq_w, + limited, + self.spatial_merge_size, + ) # Encoder expects (N, 1, hidden_size) — add batch dim hidden = hidden.unsqueeze(1) - # On ROCm, the flash_attn Triton rotary kernel can fail with - # hipErrorInvalidValue when seqlen is very large, e.g. 192k video - # tokens; pass per-segment lengths so RoPE can be applied in chunks. - # On other platforms leave it None -> single-kernel fast path, so the - # NVIDIA/CUDA code path is unchanged. - rotary_segment_lengths = lens if current_platform.is_rocm() else None hidden = self.encoder( hidden, @@ -579,7 +451,6 @@ class MiniMaxVLVisionTransformer(nn.Module): rotary_cos, rotary_sin, max_seqlen, - rotary_segment_lengths, sequence_lengths=sequence_lengths, ) hidden = hidden.squeeze(1) # back to (total_N, hidden_size)