[ROCm][DSv4.1][Perf] Fuse the inverse RoPE into the sparse decode reduce (#57435)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
72675c4706
commit
ca831d1c55
@@ -259,7 +259,10 @@ def _launch_sparse_decode_reduce(
|
||||
part_l: torch.Tensor,
|
||||
part_acc: torch.Tensor,
|
||||
adaptive_splits: bool,
|
||||
positions: torch.Tensor | None = None,
|
||||
cos_sin_cache: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run the reduce kernel, with the inverse-RoPE epilogue if given positions."""
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
num_queries, num_splits, num_heads = part_m.shape
|
||||
@@ -275,6 +278,8 @@ def _launch_sparse_decode_reduce(
|
||||
part_acc,
|
||||
attn_sink,
|
||||
out,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
part_m.stride(0),
|
||||
@@ -282,6 +287,7 @@ def _launch_sparse_decode_reduce(
|
||||
part_acc.stride(0),
|
||||
part_acc.stride(1),
|
||||
part_acc.stride(2),
|
||||
cos_sin_cache.stride(0) if cos_sin_cache is not None else 0,
|
||||
num_heads,
|
||||
HAS_ATTN_SINK=False,
|
||||
ADAPTIVE_SPLITS=adaptive_splits,
|
||||
@@ -289,6 +295,9 @@ def _launch_sparse_decode_reduce(
|
||||
BLOCK_H=1,
|
||||
NUM_SPLITS=num_splits,
|
||||
SPLITS_PAD=1 << (num_splits - 1).bit_length(),
|
||||
FUSE_INV_ROPE=positions is not None,
|
||||
NOPE=NOPE_HEAD_DIM,
|
||||
HALF=ROPE_HEAD_DIM // 2,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
@@ -1008,6 +1017,46 @@ def test_sparse_attn_decode_gfx950_adaptive_reduce_ignores_stale_scratch() -> No
|
||||
assert torch.equal(actual, torch.full_like(actual, 2))
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@pytest.mark.parametrize("adaptive_splits", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_reduce_inverse_rope_epilogue(adaptive_splits: bool) -> None:
|
||||
"""FUSE_INV_ROPE must land where the standalone rotation would.
|
||||
|
||||
The epilogue rotates a whole [H, nope + rope] row with one expression by
|
||||
feeding cos=1/sin=0 to the NoPE lanes, which only holds if the pair split
|
||||
lands on ``nope_head_dim // 2``; the NoPE lanes therefore have to come
|
||||
back untouched, not merely close.
|
||||
"""
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _fused_inverse_rope_gptj
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(7)
|
||||
num_queries, num_splits, num_heads, max_pos = 5, 4, 8, 64
|
||||
shape = (num_queries, num_splits, num_heads)
|
||||
part_m = torch.randn(shape, dtype=torch.float32, device=device)
|
||||
part_l = torch.rand(shape, dtype=torch.float32, device=device) + 0.5
|
||||
part_acc = torch.randn((*shape, HEAD_DIM), dtype=torch.float32, device=device)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_queries,), dtype=torch.int64, device=device
|
||||
)
|
||||
angle = torch.randn(max_pos, ROPE_HEAD_DIM // 2, device=device)
|
||||
cos_sin_cache = torch.cat((angle.cos(), angle.sin()), dim=-1).contiguous()
|
||||
|
||||
unfused = _launch_sparse_decode_reduce(part_m, part_l, part_acc, adaptive_splits)
|
||||
expected = _fused_inverse_rope_gptj(
|
||||
unfused, positions, cos_sin_cache, ROPE_HEAD_DIM
|
||||
)
|
||||
actual = _launch_sparse_decode_reduce(
|
||||
part_m, part_l, part_acc, adaptive_splits, positions, cos_sin_cache
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
assert torch.equal(actual[..., :NOPE_HEAD_DIM], unfused[..., :NOPE_HEAD_DIM])
|
||||
# An epilogue that quietly did nothing would satisfy everything above.
|
||||
assert not torch.equal(actual[..., NOPE_HEAD_DIM:], unfused[..., NOPE_HEAD_DIM:])
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@pytest.mark.parametrize("extra_len", [0, 1, 31, 32, 33, 63, 64, 65])
|
||||
@torch.inference_mode()
|
||||
|
||||
Reference in New Issue
Block a user