diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index 5b68732f7a..a966fb60a4 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -259,7 +259,10 @@ def _launch_sparse_decode_reduce( part_l: torch.Tensor, part_acc: torch.Tensor, adaptive_splits: bool, + positions: torch.Tensor | None = None, + cos_sin_cache: torch.Tensor | None = None, ) -> torch.Tensor: + """Run the reduce kernel, with the inverse-RoPE epilogue if given positions.""" from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod num_queries, num_splits, num_heads = part_m.shape @@ -275,6 +278,8 @@ def _launch_sparse_decode_reduce( part_acc, attn_sink, out, + positions, + cos_sin_cache, out.stride(0), out.stride(1), part_m.stride(0), @@ -282,6 +287,7 @@ def _launch_sparse_decode_reduce( part_acc.stride(0), part_acc.stride(1), part_acc.stride(2), + cos_sin_cache.stride(0) if cos_sin_cache is not None else 0, num_heads, HAS_ATTN_SINK=False, ADAPTIVE_SPLITS=adaptive_splits, @@ -289,6 +295,9 @@ def _launch_sparse_decode_reduce( BLOCK_H=1, NUM_SPLITS=num_splits, SPLITS_PAD=1 << (num_splits - 1).bit_length(), + FUSE_INV_ROPE=positions is not None, + NOPE=NOPE_HEAD_DIM, + HALF=ROPE_HEAD_DIM // 2, num_warps=4, ) return out @@ -1008,6 +1017,46 @@ def test_sparse_attn_decode_gfx950_adaptive_reduce_ignores_stale_scratch() -> No assert torch.equal(actual, torch.full_like(actual, 2)) +@requires_gfx950 +@pytest.mark.parametrize("adaptive_splits", [False, True]) +@torch.inference_mode() +def test_sparse_attn_decode_reduce_inverse_rope_epilogue(adaptive_splits: bool) -> None: + """FUSE_INV_ROPE must land where the standalone rotation would. + + The epilogue rotates a whole [H, nope + rope] row with one expression by + feeding cos=1/sin=0 to the NoPE lanes, which only holds if the pair split + lands on ``nope_head_dim // 2``; the NoPE lanes therefore have to come + back untouched, not merely close. + """ + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _fused_inverse_rope_gptj + + device = torch.device("cuda") + torch.manual_seed(7) + num_queries, num_splits, num_heads, max_pos = 5, 4, 8, 64 + shape = (num_queries, num_splits, num_heads) + part_m = torch.randn(shape, dtype=torch.float32, device=device) + part_l = torch.rand(shape, dtype=torch.float32, device=device) + 0.5 + part_acc = torch.randn((*shape, HEAD_DIM), dtype=torch.float32, device=device) + positions = torch.randint( + 0, max_pos, (num_queries,), dtype=torch.int64, device=device + ) + angle = torch.randn(max_pos, ROPE_HEAD_DIM // 2, device=device) + cos_sin_cache = torch.cat((angle.cos(), angle.sin()), dim=-1).contiguous() + + unfused = _launch_sparse_decode_reduce(part_m, part_l, part_acc, adaptive_splits) + expected = _fused_inverse_rope_gptj( + unfused, positions, cos_sin_cache, ROPE_HEAD_DIM + ) + actual = _launch_sparse_decode_reduce( + part_m, part_l, part_acc, adaptive_splits, positions, cos_sin_cache + ) + + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) + assert torch.equal(actual[..., :NOPE_HEAD_DIM], unfused[..., :NOPE_HEAD_DIM]) + # An epilogue that quietly did nothing would satisfy everything above. + assert not torch.equal(actual[..., NOPE_HEAD_DIM:], unfused[..., NOPE_HEAD_DIM:]) + + @requires_gfx950 @pytest.mark.parametrize("extra_len", [0, 1, 31, 32, 33, 63, 64, 65]) @torch.inference_mode() diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index b7a67e85bd..dee7237ff4 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -37,6 +37,7 @@ from vllm.v1.attention.backends.mla.sparse_swa import ( from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( build_ragged_indices_from_dense, rocm_inv_rope_einsum, + rocm_inverse_rope_rows_, rocm_sparse_attn_decode, rocm_sparse_attn_prefill, ) @@ -673,6 +674,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): self.n_local_groups, self.o_lora_rank, self.wo_a, + inverse_rope=False, ) zf = z.flatten(1) if self._wo_b_scale is not None and zf.dim() == 2: @@ -747,15 +749,28 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): attn_metadata=rocm_metadata, swa_metadata=swa_metadata, ) + rotated = 0 if num_decodes > 0: - self._forward_decode( + rotated = self._forward_decode( q=q[:num_decode_tokens], + positions=positions[:num_decode_tokens], kv_cache=self_kv_cache, swa_metadata=swa_metadata, attn_metadata=rocm_metadata, swa_only=swa_only, output=output[:num_decode_tokens], ) + # Only the decode reduce rotates its own rows, and only the leading + # `rotated` of them; prefill rows and any decode path that did not + # fuse still owe the standalone pass. Settle that here rather than in + # _o_proj: the split is batch-dependent and _o_proj runs compiled, + # where such a value freezes at its trace-time value. + rocm_inverse_rope_rows_( + output[rotated:, : self.n_local_heads, :], + positions[rotated:], + self.rotary_emb.cos_sin_cache, + self.rope_head_dim, + ) def _decode_topk_ragged( self, @@ -796,12 +811,14 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): def _forward_decode( self, q: torch.Tensor, + positions: torch.Tensor, kv_cache: torch.Tensor | None, swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, attn_metadata: DeepseekV4FlashMLAMetadata | None, swa_only: bool, output: torch.Tensor, - ) -> None: + ) -> int: + """Returns how many leading rows the decode epilogue inverse-RoPE'd.""" num_decodes = swa_metadata.num_decodes num_decode_tokens = swa_metadata.num_decode_tokens @@ -820,7 +837,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): num_decode_tokens=num_decode_tokens, ) - rocm_sparse_attn_decode( + return rocm_sparse_attn_decode( q=q, kv_cache=kv_cache, swa_k_cache=self.swa_cache_layer.kv_cache, @@ -839,6 +856,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): nope_head_dim=self.nope_head_dim, rope_head_dim=self.rope_head_dim, output=output, + inv_rope_positions=positions, + inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache, extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free( self.kv_cache_dtype, self._has_kv_transfer, diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 6b1530d74e..d14eedb1e1 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -1572,8 +1572,13 @@ def _fused_inverse_rope_gptj( positions: torch.Tensor, cos_sin_cache: torch.Tensor, rope_head_dim: int, + out: torch.Tensor | None = None, ) -> torch.Tensor: - """bf16 inverse GPT-J RoPE via a single fused Triton kernel.""" + """bf16 inverse GPT-J RoPE via a single fused Triton kernel. + + ``out`` may alias ``o``: the rotation is a per-row bijection whose kernel + reads both lanes of a pair before storing either. + """ assert o.dim() == 3 and o.stride(-1) == 1, ( "_fused_inverse_rope_gptj expects a [T, H, D] input with a contiguous last dim" ) @@ -1585,9 +1590,14 @@ def _fused_inverse_rope_gptj( f"[P, {rope_head_dim}] = cos | sin, got {tuple(cos_sin_cache.shape)}" ) num_tokens, num_heads, head_dim = o.shape - out = torch.empty( - (num_tokens, num_heads, head_dim), dtype=torch.bfloat16, device=o.device - ) + if out is None: + out = torch.empty( + (num_tokens, num_heads, head_dim), dtype=torch.bfloat16, device=o.device + ) + else: + assert out.dtype == torch.bfloat16, ( + f"inverse RoPE writes bf16, got an output buffer of {out.dtype}" + ) if num_tokens == 0: return out _inverse_rope_gptj_kernel[(num_tokens, num_heads)]( @@ -1608,6 +1618,25 @@ def _fused_inverse_rope_gptj( return out +def rocm_inverse_rope_rows_( + o: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + rope_head_dim: int, +) -> None: + """Inverse-RoPE attention output rows in place. + + For rows no attention kernel rotated in its epilogue. Call it from the + eager attention segment: which rows still owe a rotation depends on the + prefill/decode split, and the o_proj that used to do this runs inside the + compiled region, where a batch-dependent Python value would be frozen at + trace time. + """ + if o.shape[0] == 0: + return + _fused_inverse_rope_gptj(o, positions, cos_sin_cache, rope_head_dim, out=o) + + def _get_cached_wo_a_bf16( wo_a: torch.nn.Module, n_local_groups: int, @@ -1664,16 +1693,27 @@ def rocm_inv_rope_einsum( n_local_groups: int, o_lora_rank: int, wo_a: torch.nn.Module, + inverse_rope: bool = True, ) -> torch.Tensor: """Inverse-RoPE + WO_A bmm path used on ROCm. Fuses the inverse GPT-J RoPE into one Triton kernel and caches the bf16 - wo_a weight so the per-step dequant disappears. + wo_a weight so the per-step dequant disappears. Callers whose attention + already rotated every row pass ``inverse_rope=False``; that is a property + of the attention backend, not of the batch, so it stays constant across + steps and is safe to read from compiled code. """ - o_ref = _fused_inverse_rope_gptj( - o, positions, rotary_emb.cos_sin_cache, rope_head_dim - ) - o_ref = o_ref.view(o.shape[0], n_local_groups, -1) + if inverse_rope: + o_ref = _fused_inverse_rope_gptj( + o, positions, rotary_emb.cos_sin_cache, rope_head_dim + ) + else: + assert o.dtype == torch.bfloat16, ( + "a pre-rotated attention output feeds the wo_a bmm directly, so it " + f"must already be bf16, got {o.dtype}" + ) + o_ref = o + o_ref = o_ref.reshape(o.shape[0], n_local_groups, -1) wo_a_weight = _get_cached_wo_a_bf16( wo_a, n_local_groups, o_lora_rank, o_ref.shape[-1] @@ -2889,6 +2929,8 @@ def _sparse_attn_decode_reduce_kernel( part_acc_ptr, attn_sink_ptr, out_ptr, + pos_ptr, + cos_sin_ptr, out_stride0, out_stride1, pm_stride0, @@ -2896,6 +2938,7 @@ def _sparse_attn_decode_reduce_kernel( pa_stride0, pa_stride_s, pa_stride_h, + cs_stride, num_heads, HAS_ATTN_SINK: tl.constexpr, ADAPTIVE_SPLITS: tl.constexpr, @@ -2903,6 +2946,9 @@ def _sparse_attn_decode_reduce_kernel( BLOCK_H: tl.constexpr, NUM_SPLITS: tl.constexpr, SPLITS_PAD: tl.constexpr, + FUSE_INV_ROPE: tl.constexpr, + NOPE: tl.constexpr, + HALF: tl.constexpr, ): query_idx = tl.program_id(0) pid_h = tl.program_id(1) @@ -2992,6 +3038,26 @@ def _sparse_attn_decode_reduce_kernel( out = tl.where(l_final[:, None] > 0.0, acc / denom[:, None], 0.0) + if FUSE_INV_ROPE: + # Inverse GPT-J RoPE on the trailing rope lanes, straight out of the + # combine registers: out_even = a*cos + b*sin, out_odd = b*cos - a*sin. + # NoPE lanes take cos=1/sin=0 so one expression covers the whole row + # and the o_proj rotation pass disappears for these tokens. + pos = tl.load(pos_ptr + query_idx).to(tl.int64) + pair_idx = tl.arange(0, COMB_DIM // 2) - (NOPE // 2) + is_rope = pair_idx >= 0 + k = tl.where(is_rope, pair_idx, 0) + cos = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + k), 1.0) + sin = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + HALF + k), 0.0) + even, odd = tl.split(tl.reshape(out, (BLOCK_H, COMB_DIM // 2, 2))) + out = tl.reshape( + tl.join( + even * cos[None, :] + odd * sin[None, :], + odd * cos[None, :] - even * sin[None, :], + ), + (BLOCK_H, COMB_DIM), + ) + out_row_ptr = ( out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1 ) @@ -3312,6 +3378,8 @@ def _rocm_sparse_attn_decode_ragged_triton( out: torch.Tensor | None = None, extra_cache_nan_free: bool = False, adaptive_splits: bool = False, + inv_rope_positions: torch.Tensor | None = None, + inv_rope_cos_sin_cache: torch.Tensor | None = None, ) -> torch.Tensor: assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}" assert main_cache.ndim == 3, ( @@ -3554,12 +3622,27 @@ def _rocm_sparse_attn_decode_ragged_triton( num_warps=4, ) + fuse_inv_rope = inv_rope_positions is not None + if fuse_inv_rope: + assert inv_rope_cos_sin_cache is not None + assert inv_rope_cos_sin_cache.shape[-1] == rope_head_dim, ( + "fused inverse RoPE expects cos_sin_cache laid out as " + f"[P, {rope_head_dim}] = cos | sin, got " + f"{tuple(inv_rope_cos_sin_cache.shape)}" + ) + assert nope_head_dim % 2 == 0 and rope_head_dim % 2 == 0, ( + "fused inverse RoPE pairs adjacent lanes, so both head dims must " + f"be even, got nope={nope_head_dim} rope={rope_head_dim}" + ) + _sparse_attn_decode_reduce_kernel[(num_queries, num_heads)]( part_m, part_l, part_acc, attn_sink, out, + inv_rope_positions, + inv_rope_cos_sin_cache, out.stride(0), out.stride(1), part_m.stride(0), @@ -3567,6 +3650,7 @@ def _rocm_sparse_attn_decode_ragged_triton( part_acc.stride(0), part_acc.stride(1), part_acc.stride(2), + inv_rope_cos_sin_cache.stride(0) if inv_rope_cos_sin_cache is not None else 0, num_heads, HAS_ATTN_SINK=has_attn_sink, ADAPTIVE_SPLITS=adaptive_splits, @@ -3574,6 +3658,9 @@ def _rocm_sparse_attn_decode_ragged_triton( BLOCK_H=1, NUM_SPLITS=num_splits, SPLITS_PAD=triton.next_power_of_2(num_splits), + FUSE_INV_ROPE=fuse_inv_rope, + NOPE=nope_head_dim, + HALF=rope_head_dim // 2, num_warps=4, ) return out @@ -3598,6 +3685,8 @@ def _rocm_sparse_attn_decode_triton( out: torch.Tensor | None = None, extra_cache_nan_free: bool = False, adaptive_splits: bool = False, + inv_rope_positions: torch.Tensor | None = None, + inv_rope_cos_sin_cache: torch.Tensor | None = None, ) -> torch.Tensor: if main_ragged_indices is None or main_ragged_indptr is None: main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense( @@ -3636,6 +3725,8 @@ def _rocm_sparse_attn_decode_triton( out=out, extra_cache_nan_free=extra_cache_nan_free, adaptive_splits=adaptive_splits, + inv_rope_positions=inv_rope_positions, + inv_rope_cos_sin_cache=inv_rope_cos_sin_cache, ) @@ -3737,7 +3828,16 @@ def rocm_sparse_attn_decode( output: torch.Tensor, extra_cache_nan_free: bool = False, adaptive_splits: bool = False, -) -> None: + inv_rope_positions: torch.Tensor | None = None, + inv_rope_cos_sin_cache: torch.Tensor | None = None, +) -> int: + """Run sparse MLA decode into ``output``. + + Passing ``inv_rope_positions`` folds the inverse RoPE into the reduce + epilogue. Returns how many leading rows of ``output`` came back rotated, + so a caller mixing in a decode path that does not fuse still knows what it + owes the standalone pass. Read it from the eager attention segment only. + """ assert swa_k_cache.dtype == torch.uint8, ( "ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, " f"got {swa_k_cache.dtype}" @@ -3786,6 +3886,9 @@ def rocm_sparse_attn_decode( out=direct_out, extra_cache_nan_free=extra_cache_nan_free, adaptive_splits=adaptive_splits, + inv_rope_positions=inv_rope_positions, + inv_rope_cos_sin_cache=inv_rope_cos_sin_cache, ) if direct_out is None: output.copy_(attn_out.to(output.dtype)) + return output.shape[0] if inv_rope_positions is not None else 0