From 05a08148631ab2d852c1984ddb3abebad8039a29 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 28 Jul 2026 11:50:10 -0500 Subject: [PATCH] [ROCm] Fix and optimize GPT-J-style MRoPE (#49906) Signed-off-by: Andreas Karatzas --- tests/kernels/core/test_mrope.py | 10 +- .../layers/rotary_embedding/mrope.py | 149 +++++++++++------- 2 files changed, 100 insertions(+), 59 deletions(-) diff --git a/tests/kernels/core/test_mrope.py b/tests/kernels/core/test_mrope.py index 29051b4a00..6f64fabfe9 100644 --- a/tests/kernels/core/test_mrope.py +++ b/tests/kernels/core/test_mrope.py @@ -38,6 +38,7 @@ def generate_test_data( class MRoPETestInfo(NamedTuple): model_name: str + is_neox_style: bool = True # https://github.com/pytorch/pytorch/blob/main/torch/testing/_comparison.py#L1317 atol: float = 1e-2 rtol: float = 1.6e-2 @@ -45,7 +46,10 @@ class MRoPETestInfo(NamedTuple): MODELS_TO_TEST = [ - MRoPETestInfo(model_name="zai-org/GLM-4.1V-9B-Thinking"), + MRoPETestInfo( + model_name="zai-org/GLM-4.1V-9B-Thinking", + is_neox_style=False, + ), MRoPETestInfo(model_name="Qwen/Qwen2-VL-7B-Instruct"), MRoPETestInfo(model_name="Qwen/Qwen2-VL-72B-Instruct"), MRoPETestInfo(model_name="Qwen/Qwen2.5-VL-72B-Instruct"), @@ -92,7 +96,7 @@ def test_mrope( if hasattr(config, "head_dim") else config.hidden_size // total_num_heads ) - is_neox_style = True + is_neox_style = model_info.is_neox_style max_position = config.max_position_embeddings @@ -162,7 +166,7 @@ def test_mrope_torch_compile_tracing( if hasattr(config, "head_dim") else config.hidden_size // total_num_heads ) - is_neox_style = True + is_neox_style = model_info.is_neox_style max_position = config.max_position_embeddings mrope_helper_class = get_rope( diff --git a/vllm/model_executor/layers/rotary_embedding/mrope.py b/vllm/model_executor/layers/rotary_embedding/mrope.py index 3c946dd130..29ce9e5000 100644 --- a/vllm/model_executor/layers/rotary_embedding/mrope.py +++ b/vllm/model_executor/layers/rotary_embedding/mrope.py @@ -5,6 +5,7 @@ import numpy as np import torch +from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from .base import RotaryEmbeddingBase @@ -24,16 +25,17 @@ def _triton_mrope_forward( rd: tl.constexpr, pad_n_qh: tl.constexpr, pad_n_kh: tl.constexpr, - pad_hd: tl.constexpr, + pad_rd: tl.constexpr, mrope_section_t: tl.constexpr, mrope_section_h: tl.constexpr, mrope_section_w: tl.constexpr, is_interleaved: tl.constexpr, + is_neox_style: tl.constexpr, ): # Adapted from # https://github.com/linkedin/Liger-Kernel/blob/main/src/liger_kernel/ops/qwen2vl_mrope.py # This version supports flatten input tensors from vllm - # and supports cos and sin cache with shape (3, num_tokens, head_dim // 2) + # and supports cos and sin cache with shape (3, num_tokens, rotary_dim // 2) # instead of (3, bsz, seq_len, head_dim), also supports interleaved rotary pid = tl.program_id(0) # locate start address @@ -44,9 +46,9 @@ def _triton_mrope_forward( # get the cos(mθ_{i...d/2}) and sin(mθ_{i...d/2}) for token position # m of this program instance # #################################################################### - # Note: cos and sin now have shape (3, num_tokens, head_dim // 2) + # Note: cos and sin now have shape (3, num_tokens, rotary_dim // 2) - # Updated stride calculation for half head_dim + # Updated stride calculation for half rotary_dim half_rd = rd // 2 t_cos = cos + pid * half_rd h_cos = t_cos + num_tokens * half_rd @@ -55,12 +57,17 @@ def _triton_mrope_forward( h_sin = t_sin + num_tokens * half_rd w_sin = h_sin + num_tokens * half_rd - # Updated offsets for half head_dim - cos_offsets = tl.arange(0, pad_hd // 2) + # Updated offsets for half rotary_dim + cos_offsets = tl.arange(0, pad_rd // 2) if is_interleaved: - h_mask = ((cos_offsets % 3) == 1) & (cos_offsets <= 3 * mrope_section_h) - w_mask = ((cos_offsets % 3) == 2) & (cos_offsets <= 3 * mrope_section_w) - t_mask = ~(h_mask | w_mask) + valid_mask = cos_offsets < half_rd + h_mask = ( + valid_mask & ((cos_offsets % 3) == 1) & (cos_offsets <= 3 * mrope_section_h) + ) + w_mask = ( + valid_mask & ((cos_offsets % 3) == 2) & (cos_offsets <= 3 * mrope_section_w) + ) + t_mask = valid_mask & ~(h_mask | w_mask) else: t_end = mrope_section_t h_end = t_end + mrope_section_h @@ -79,55 +86,74 @@ def _triton_mrope_forward( sin_row = t_sin_row + h_sin_row + w_sin_row # #################################################################### - # Load the left and right half of q and k for the current - # program instance (i.e. for the current token) separately + # Load the two values in each rotary pair for the current token. + # NeoX pairs the first and second halves, while GPT-J pairs + # adjacent values. # #################################################################### - # left half of the head - first_half_q_offsets = ( - tl.arange(0, pad_n_qh)[:, None] * hd + tl.arange(0, pad_hd // 2)[None, :] - ) - first_half_k_offsets = ( - tl.arange(0, pad_n_kh)[:, None] * hd + tl.arange(0, pad_hd // 2)[None, :] - ) - first_q_mask = (tl.arange(0, pad_n_qh)[:, None] < n_qh) & ( - tl.arange(0, pad_hd // 2)[None, :] < rd // 2 - ) - first_k_mask = (tl.arange(0, pad_n_kh)[:, None] < n_kh) & ( - tl.arange(0, pad_hd // 2)[None, :] < rd // 2 - ) + if is_neox_style: + rotary_offsets = tl.arange(0, pad_rd // 2) + first_q_offsets = tl.arange(0, pad_n_qh)[:, None] * hd + rotary_offsets[None, :] + first_k_offsets = tl.arange(0, pad_n_kh)[:, None] * hd + rotary_offsets[None, :] + first_q_mask = (tl.arange(0, pad_n_qh)[:, None] < n_qh) & ( + rotary_offsets[None, :] < rd // 2 + ) + first_k_mask = (tl.arange(0, pad_n_kh)[:, None] < n_kh) & ( + rotary_offsets[None, :] < rd // 2 + ) - q_tile_1 = tl.load(q_ptr + first_half_q_offsets, mask=first_q_mask, other=0).to( - sin_row.dtype - ) - k_tile_1 = tl.load(k_ptr + first_half_k_offsets, mask=first_k_mask, other=0).to( - sin_row.dtype - ) + q_tile_1 = tl.load(q_ptr + first_q_offsets, mask=first_q_mask, other=0).to( + sin_row.dtype + ) + k_tile_1 = tl.load(k_ptr + first_k_offsets, mask=first_k_mask, other=0).to( + sin_row.dtype + ) - # right half of the head - second_half_q_offsets = first_half_q_offsets + (rd // 2) - second_half_k_offsets = first_half_k_offsets + (rd // 2) - second_q_mask = first_q_mask - second_k_mask = first_k_mask + second_q_offsets = first_q_offsets + (rd // 2) + second_k_offsets = first_k_offsets + (rd // 2) + q_tile_2 = tl.load(q_ptr + second_q_offsets, mask=first_q_mask, other=0).to( + sin_row.dtype + ) + k_tile_2 = tl.load(k_ptr + second_k_offsets, mask=first_k_mask, other=0).to( + sin_row.dtype + ) - q_tile_2 = tl.load(q_ptr + second_half_q_offsets, mask=second_q_mask, other=0).to( - sin_row.dtype - ) - k_tile_2 = tl.load(k_ptr + second_half_k_offsets, mask=second_k_mask, other=0).to( - sin_row.dtype - ) + new_q_tile_1 = q_tile_1 * cos_row - q_tile_2 * sin_row + tl.store(q_ptr + first_q_offsets, new_q_tile_1, mask=first_q_mask) + new_q_tile_2 = q_tile_2 * cos_row + q_tile_1 * sin_row + tl.store(q_ptr + second_q_offsets, new_q_tile_2, mask=first_q_mask) - # y = [x1, x2] * [cos, cos] + [-x2, x1] * [sin, sin] - # Since cos and sin are now half-size, - # we use the same cos_row and sin_row for both halves - new_q_tile_1 = q_tile_1 * cos_row - q_tile_2 * sin_row - tl.store(q_ptr + first_half_q_offsets, new_q_tile_1, mask=first_q_mask) - new_q_tile_2 = q_tile_2 * cos_row + q_tile_1 * sin_row - tl.store(q_ptr + second_half_q_offsets, new_q_tile_2, mask=second_q_mask) + new_k_tile_1 = k_tile_1 * cos_row - k_tile_2 * sin_row + tl.store(k_ptr + first_k_offsets, new_k_tile_1, mask=first_k_mask) + new_k_tile_2 = k_tile_2 * cos_row + k_tile_1 * sin_row + tl.store(k_ptr + second_k_offsets, new_k_tile_2, mask=first_k_mask) + else: + # Load and store adjacent rotary pairs contiguously. Using stride-two + # even/odd offsets makes Triton emit scalar 16-bit memory operations on + # AMD, while split/interleave only rearranges values in registers. + rotary_offsets = tl.arange(0, pad_rd) + q_offsets = tl.arange(0, pad_n_qh)[:, None] * hd + rotary_offsets[None, :] + k_offsets = tl.arange(0, pad_n_kh)[:, None] * hd + rotary_offsets[None, :] + q_mask = (tl.arange(0, pad_n_qh)[:, None] < n_qh) & ( + rotary_offsets[None, :] < rd + ) + k_mask = (tl.arange(0, pad_n_kh)[:, None] < n_kh) & ( + rotary_offsets[None, :] < rd + ) - new_k_tile_1 = k_tile_1 * cos_row - k_tile_2 * sin_row - tl.store(k_ptr + first_half_k_offsets, new_k_tile_1, mask=first_k_mask) - new_k_tile_2 = k_tile_2 * cos_row + k_tile_1 * sin_row - tl.store(k_ptr + second_half_k_offsets, new_k_tile_2, mask=second_k_mask) + q_tile = tl.load(q_ptr + q_offsets, mask=q_mask, other=0).to(sin_row.dtype) + k_tile = tl.load(k_ptr + k_offsets, mask=k_mask, other=0).to(sin_row.dtype) + q_tile_1, q_tile_2 = tl.split(tl.reshape(q_tile, (pad_n_qh, pad_rd // 2, 2))) + k_tile_1, k_tile_2 = tl.split(tl.reshape(k_tile, (pad_n_kh, pad_rd // 2, 2))) + + new_q_tile_1 = q_tile_1 * cos_row - q_tile_2 * sin_row + new_q_tile_2 = q_tile_2 * cos_row + q_tile_1 * sin_row + new_q_tile = tl.interleave(new_q_tile_1, new_q_tile_2) + tl.store(q_ptr + q_offsets, new_q_tile, mask=q_mask) + + new_k_tile_1 = k_tile_1 * cos_row - k_tile_2 * sin_row + new_k_tile_2 = k_tile_2 * cos_row + k_tile_1 * sin_row + new_k_tile = tl.interleave(new_k_tile_1, new_k_tile_2) + tl.store(k_ptr + k_offsets, new_k_tile, mask=k_mask) def triton_mrope( @@ -139,23 +165,26 @@ def triton_mrope( head_size: int, rotary_dim: int, mrope_interleaved: bool, + is_neox_style: bool, ) -> tuple[torch.Tensor, torch.Tensor]: """Qwen2VL mrope kernel. Args: q: [num_tokens, num_heads * head_size] k: [num_tokens, num_kv_heads * head_size] - cos: [3, num_tokens, head_size //2 ] + cos: [3, num_tokens, rotary_dim // 2] (T/H/W positions with multimodal inputs) - sin: [3, num_tokens, head_size //2 ] + sin: [3, num_tokens, rotary_dim // 2] (T/H/W positions with multimodal inputs) mrope_section: [t, h, w] head_size: int + is_neox_style: Whether rotary pairs use split-half (NeoX) or + adjacent (GPT-J) layout. """ n_row, n_q_head_head_dim = q.shape n_q_head = n_q_head_head_dim // head_size n_kv_head = k.shape[1] // head_size - pad_hd = triton.next_power_of_2(head_size) + pad_rd = triton.next_power_of_2(rotary_dim) pad_n_q_head = triton.next_power_of_2(n_q_head) pad_n_kv_head = triton.next_power_of_2(n_kv_head) @@ -166,6 +195,11 @@ def triton_mrope( cos = cos.contiguous() sin = sin.contiguous() + # Small adjacent-pair tiles perform best with one wave per program on + # ROCm. Keep the existing launch shape for larger rotary dimensions, + # NeoX, and other backends. + use_single_wave = current_platform.is_rocm() and not is_neox_style and pad_rd <= 64 + num_warps = 1 if use_single_wave else 4 _triton_mrope_forward[(n_row,)]( q, k, @@ -178,11 +212,13 @@ def triton_mrope( rotary_dim, pad_n_q_head, pad_n_kv_head, - pad_hd, + pad_rd, mrope_section[0], mrope_section[1], mrope_section[2], mrope_interleaved, + is_neox_style, + num_warps=num_warps, ) return q, k @@ -349,6 +385,7 @@ class MRotaryEmbedding(RotaryEmbeddingBase): self.head_size, self.rotary_dim, self.mrope_interleaved, + self.is_neox_style, ) return q.reshape(query_shape), k.reshape(key_shape)