[ROCm][DSv4.1][Perf] Fuse the inverse RoPE into the sparse decode reduce (#57435)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Fangzhou Ai
2026-09-22 18:05:11 +00:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 72675c4706
commit ca831d1c55
3 changed files with 184 additions and 13 deletions
@@ -259,7 +259,10 @@ def _launch_sparse_decode_reduce(
part_l: torch.Tensor,
part_acc: torch.Tensor,
adaptive_splits: bool,
positions: torch.Tensor | None = None,
cos_sin_cache: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run the reduce kernel, with the inverse-RoPE epilogue if given positions."""
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
num_queries, num_splits, num_heads = part_m.shape
@@ -275,6 +278,8 @@ def _launch_sparse_decode_reduce(
part_acc,
attn_sink,
out,
positions,
cos_sin_cache,
out.stride(0),
out.stride(1),
part_m.stride(0),
@@ -282,6 +287,7 @@ def _launch_sparse_decode_reduce(
part_acc.stride(0),
part_acc.stride(1),
part_acc.stride(2),
cos_sin_cache.stride(0) if cos_sin_cache is not None else 0,
num_heads,
HAS_ATTN_SINK=False,
ADAPTIVE_SPLITS=adaptive_splits,
@@ -289,6 +295,9 @@ def _launch_sparse_decode_reduce(
BLOCK_H=1,
NUM_SPLITS=num_splits,
SPLITS_PAD=1 << (num_splits - 1).bit_length(),
FUSE_INV_ROPE=positions is not None,
NOPE=NOPE_HEAD_DIM,
HALF=ROPE_HEAD_DIM // 2,
num_warps=4,
)
return out
@@ -1008,6 +1017,46 @@ def test_sparse_attn_decode_gfx950_adaptive_reduce_ignores_stale_scratch() -> No
assert torch.equal(actual, torch.full_like(actual, 2))
@requires_gfx950
@pytest.mark.parametrize("adaptive_splits", [False, True])
@torch.inference_mode()
def test_sparse_attn_decode_reduce_inverse_rope_epilogue(adaptive_splits: bool) -> None:
"""FUSE_INV_ROPE must land where the standalone rotation would.
The epilogue rotates a whole [H, nope + rope] row with one expression by
feeding cos=1/sin=0 to the NoPE lanes, which only holds if the pair split
lands on ``nope_head_dim // 2``; the NoPE lanes therefore have to come
back untouched, not merely close.
"""
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _fused_inverse_rope_gptj
device = torch.device("cuda")
torch.manual_seed(7)
num_queries, num_splits, num_heads, max_pos = 5, 4, 8, 64
shape = (num_queries, num_splits, num_heads)
part_m = torch.randn(shape, dtype=torch.float32, device=device)
part_l = torch.rand(shape, dtype=torch.float32, device=device) + 0.5
part_acc = torch.randn((*shape, HEAD_DIM), dtype=torch.float32, device=device)
positions = torch.randint(
0, max_pos, (num_queries,), dtype=torch.int64, device=device
)
angle = torch.randn(max_pos, ROPE_HEAD_DIM // 2, device=device)
cos_sin_cache = torch.cat((angle.cos(), angle.sin()), dim=-1).contiguous()
unfused = _launch_sparse_decode_reduce(part_m, part_l, part_acc, adaptive_splits)
expected = _fused_inverse_rope_gptj(
unfused, positions, cos_sin_cache, ROPE_HEAD_DIM
)
actual = _launch_sparse_decode_reduce(
part_m, part_l, part_acc, adaptive_splits, positions, cos_sin_cache
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
assert torch.equal(actual[..., :NOPE_HEAD_DIM], unfused[..., :NOPE_HEAD_DIM])
# An epilogue that quietly did nothing would satisfy everything above.
assert not torch.equal(actual[..., NOPE_HEAD_DIM:], unfused[..., NOPE_HEAD_DIM:])
@requires_gfx950
@pytest.mark.parametrize("extra_len", [0, 1, 31, 32, 33, 63, 64, 65])
@torch.inference_mode()
+22 -3
View File
@@ -37,6 +37,7 @@ from vllm.v1.attention.backends.mla.sparse_swa import (
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
build_ragged_indices_from_dense,
rocm_inv_rope_einsum,
rocm_inverse_rope_rows_,
rocm_sparse_attn_decode,
rocm_sparse_attn_prefill,
)
@@ -673,6 +674,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
self.n_local_groups,
self.o_lora_rank,
self.wo_a,
inverse_rope=False,
)
zf = z.flatten(1)
if self._wo_b_scale is not None and zf.dim() == 2:
@@ -747,15 +749,28 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
attn_metadata=rocm_metadata,
swa_metadata=swa_metadata,
)
rotated = 0
if num_decodes > 0:
self._forward_decode(
rotated = self._forward_decode(
q=q[:num_decode_tokens],
positions=positions[:num_decode_tokens],
kv_cache=self_kv_cache,
swa_metadata=swa_metadata,
attn_metadata=rocm_metadata,
swa_only=swa_only,
output=output[:num_decode_tokens],
)
# Only the decode reduce rotates its own rows, and only the leading
# `rotated` of them; prefill rows and any decode path that did not
# fuse still owe the standalone pass. Settle that here rather than in
# _o_proj: the split is batch-dependent and _o_proj runs compiled,
# where such a value freezes at its trace-time value.
rocm_inverse_rope_rows_(
output[rotated:, : self.n_local_heads, :],
positions[rotated:],
self.rotary_emb.cos_sin_cache,
self.rope_head_dim,
)
def _decode_topk_ragged(
self,
@@ -796,12 +811,14 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
def _forward_decode(
self,
q: torch.Tensor,
positions: torch.Tensor,
kv_cache: torch.Tensor | None,
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_only: bool,
output: torch.Tensor,
) -> None:
) -> int:
"""Returns how many leading rows the decode epilogue inverse-RoPE'd."""
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
@@ -820,7 +837,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
num_decode_tokens=num_decode_tokens,
)
rocm_sparse_attn_decode(
return rocm_sparse_attn_decode(
q=q,
kv_cache=kv_cache,
swa_k_cache=self.swa_cache_layer.kv_cache,
@@ -839,6 +856,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
nope_head_dim=self.nope_head_dim,
rope_head_dim=self.rope_head_dim,
output=output,
inv_rope_positions=positions,
inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache,
extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free(
self.kv_cache_dtype,
self._has_kv_transfer,
+113 -10
View File
@@ -1572,8 +1572,13 @@ def _fused_inverse_rope_gptj(
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
rope_head_dim: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""bf16 inverse GPT-J RoPE via a single fused Triton kernel."""
"""bf16 inverse GPT-J RoPE via a single fused Triton kernel.
``out`` may alias ``o``: the rotation is a per-row bijection whose kernel
reads both lanes of a pair before storing either.
"""
assert o.dim() == 3 and o.stride(-1) == 1, (
"_fused_inverse_rope_gptj expects a [T, H, D] input with a contiguous last dim"
)
@@ -1585,9 +1590,14 @@ def _fused_inverse_rope_gptj(
f"[P, {rope_head_dim}] = cos | sin, got {tuple(cos_sin_cache.shape)}"
)
num_tokens, num_heads, head_dim = o.shape
out = torch.empty(
(num_tokens, num_heads, head_dim), dtype=torch.bfloat16, device=o.device
)
if out is None:
out = torch.empty(
(num_tokens, num_heads, head_dim), dtype=torch.bfloat16, device=o.device
)
else:
assert out.dtype == torch.bfloat16, (
f"inverse RoPE writes bf16, got an output buffer of {out.dtype}"
)
if num_tokens == 0:
return out
_inverse_rope_gptj_kernel[(num_tokens, num_heads)](
@@ -1608,6 +1618,25 @@ def _fused_inverse_rope_gptj(
return out
def rocm_inverse_rope_rows_(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
rope_head_dim: int,
) -> None:
"""Inverse-RoPE attention output rows in place.
For rows no attention kernel rotated in its epilogue. Call it from the
eager attention segment: which rows still owe a rotation depends on the
prefill/decode split, and the o_proj that used to do this runs inside the
compiled region, where a batch-dependent Python value would be frozen at
trace time.
"""
if o.shape[0] == 0:
return
_fused_inverse_rope_gptj(o, positions, cos_sin_cache, rope_head_dim, out=o)
def _get_cached_wo_a_bf16(
wo_a: torch.nn.Module,
n_local_groups: int,
@@ -1664,16 +1693,27 @@ def rocm_inv_rope_einsum(
n_local_groups: int,
o_lora_rank: int,
wo_a: torch.nn.Module,
inverse_rope: bool = True,
) -> torch.Tensor:
"""Inverse-RoPE + WO_A bmm path used on ROCm.
Fuses the inverse GPT-J RoPE into one Triton kernel and caches the bf16
wo_a weight so the per-step dequant disappears.
wo_a weight so the per-step dequant disappears. Callers whose attention
already rotated every row pass ``inverse_rope=False``; that is a property
of the attention backend, not of the batch, so it stays constant across
steps and is safe to read from compiled code.
"""
o_ref = _fused_inverse_rope_gptj(
o, positions, rotary_emb.cos_sin_cache, rope_head_dim
)
o_ref = o_ref.view(o.shape[0], n_local_groups, -1)
if inverse_rope:
o_ref = _fused_inverse_rope_gptj(
o, positions, rotary_emb.cos_sin_cache, rope_head_dim
)
else:
assert o.dtype == torch.bfloat16, (
"a pre-rotated attention output feeds the wo_a bmm directly, so it "
f"must already be bf16, got {o.dtype}"
)
o_ref = o
o_ref = o_ref.reshape(o.shape[0], n_local_groups, -1)
wo_a_weight = _get_cached_wo_a_bf16(
wo_a, n_local_groups, o_lora_rank, o_ref.shape[-1]
@@ -2889,6 +2929,8 @@ def _sparse_attn_decode_reduce_kernel(
part_acc_ptr,
attn_sink_ptr,
out_ptr,
pos_ptr,
cos_sin_ptr,
out_stride0,
out_stride1,
pm_stride0,
@@ -2896,6 +2938,7 @@ def _sparse_attn_decode_reduce_kernel(
pa_stride0,
pa_stride_s,
pa_stride_h,
cs_stride,
num_heads,
HAS_ATTN_SINK: tl.constexpr,
ADAPTIVE_SPLITS: tl.constexpr,
@@ -2903,6 +2946,9 @@ def _sparse_attn_decode_reduce_kernel(
BLOCK_H: tl.constexpr,
NUM_SPLITS: tl.constexpr,
SPLITS_PAD: tl.constexpr,
FUSE_INV_ROPE: tl.constexpr,
NOPE: tl.constexpr,
HALF: tl.constexpr,
):
query_idx = tl.program_id(0)
pid_h = tl.program_id(1)
@@ -2992,6 +3038,26 @@ def _sparse_attn_decode_reduce_kernel(
out = tl.where(l_final[:, None] > 0.0, acc / denom[:, None], 0.0)
if FUSE_INV_ROPE:
# Inverse GPT-J RoPE on the trailing rope lanes, straight out of the
# combine registers: out_even = a*cos + b*sin, out_odd = b*cos - a*sin.
# NoPE lanes take cos=1/sin=0 so one expression covers the whole row
# and the o_proj rotation pass disappears for these tokens.
pos = tl.load(pos_ptr + query_idx).to(tl.int64)
pair_idx = tl.arange(0, COMB_DIM // 2) - (NOPE // 2)
is_rope = pair_idx >= 0
k = tl.where(is_rope, pair_idx, 0)
cos = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + k), 1.0)
sin = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + HALF + k), 0.0)
even, odd = tl.split(tl.reshape(out, (BLOCK_H, COMB_DIM // 2, 2)))
out = tl.reshape(
tl.join(
even * cos[None, :] + odd * sin[None, :],
odd * cos[None, :] - even * sin[None, :],
),
(BLOCK_H, COMB_DIM),
)
out_row_ptr = (
out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1
)
@@ -3312,6 +3378,8 @@ def _rocm_sparse_attn_decode_ragged_triton(
out: torch.Tensor | None = None,
extra_cache_nan_free: bool = False,
adaptive_splits: bool = False,
inv_rope_positions: torch.Tensor | None = None,
inv_rope_cos_sin_cache: torch.Tensor | None = None,
) -> torch.Tensor:
assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}"
assert main_cache.ndim == 3, (
@@ -3554,12 +3622,27 @@ def _rocm_sparse_attn_decode_ragged_triton(
num_warps=4,
)
fuse_inv_rope = inv_rope_positions is not None
if fuse_inv_rope:
assert inv_rope_cos_sin_cache is not None
assert inv_rope_cos_sin_cache.shape[-1] == rope_head_dim, (
"fused inverse RoPE expects cos_sin_cache laid out as "
f"[P, {rope_head_dim}] = cos | sin, got "
f"{tuple(inv_rope_cos_sin_cache.shape)}"
)
assert nope_head_dim % 2 == 0 and rope_head_dim % 2 == 0, (
"fused inverse RoPE pairs adjacent lanes, so both head dims must "
f"be even, got nope={nope_head_dim} rope={rope_head_dim}"
)
_sparse_attn_decode_reduce_kernel[(num_queries, num_heads)](
part_m,
part_l,
part_acc,
attn_sink,
out,
inv_rope_positions,
inv_rope_cos_sin_cache,
out.stride(0),
out.stride(1),
part_m.stride(0),
@@ -3567,6 +3650,7 @@ def _rocm_sparse_attn_decode_ragged_triton(
part_acc.stride(0),
part_acc.stride(1),
part_acc.stride(2),
inv_rope_cos_sin_cache.stride(0) if inv_rope_cos_sin_cache is not None else 0,
num_heads,
HAS_ATTN_SINK=has_attn_sink,
ADAPTIVE_SPLITS=adaptive_splits,
@@ -3574,6 +3658,9 @@ def _rocm_sparse_attn_decode_ragged_triton(
BLOCK_H=1,
NUM_SPLITS=num_splits,
SPLITS_PAD=triton.next_power_of_2(num_splits),
FUSE_INV_ROPE=fuse_inv_rope,
NOPE=nope_head_dim,
HALF=rope_head_dim // 2,
num_warps=4,
)
return out
@@ -3598,6 +3685,8 @@ def _rocm_sparse_attn_decode_triton(
out: torch.Tensor | None = None,
extra_cache_nan_free: bool = False,
adaptive_splits: bool = False,
inv_rope_positions: torch.Tensor | None = None,
inv_rope_cos_sin_cache: torch.Tensor | None = None,
) -> torch.Tensor:
if main_ragged_indices is None or main_ragged_indptr is None:
main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense(
@@ -3636,6 +3725,8 @@ def _rocm_sparse_attn_decode_triton(
out=out,
extra_cache_nan_free=extra_cache_nan_free,
adaptive_splits=adaptive_splits,
inv_rope_positions=inv_rope_positions,
inv_rope_cos_sin_cache=inv_rope_cos_sin_cache,
)
@@ -3737,7 +3828,16 @@ def rocm_sparse_attn_decode(
output: torch.Tensor,
extra_cache_nan_free: bool = False,
adaptive_splits: bool = False,
) -> None:
inv_rope_positions: torch.Tensor | None = None,
inv_rope_cos_sin_cache: torch.Tensor | None = None,
) -> int:
"""Run sparse MLA decode into ``output``.
Passing ``inv_rope_positions`` folds the inverse RoPE into the reduce
epilogue. Returns how many leading rows of ``output`` came back rotated,
so a caller mixing in a decode path that does not fuse still knows what it
owes the standalone pass. Read it from the eager attention segment only.
"""
assert swa_k_cache.dtype == torch.uint8, (
"ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, "
f"got {swa_k_cache.dtype}"
@@ -3786,6 +3886,9 @@ def rocm_sparse_attn_decode(
out=direct_out,
extra_cache_nan_free=extra_cache_nan_free,
adaptive_splits=adaptive_splits,
inv_rope_positions=inv_rope_positions,
inv_rope_cos_sin_cache=inv_rope_cos_sin_cache,
)
if direct_out is None:
output.copy_(attn_out.to(output.dtype))
return output.shape[0] if inv_rope_positions is not None else 0