[ROCm][Perf][GLM-5.3-Flash] Remove redundant copy after ragged sparse MLA (#58569)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Simon Danielsson
2026-10-02 11:45:51 +01:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5688a4dd4a
commit 751fa96a49
2 changed files with 130 additions and 7 deletions
@@ -596,6 +596,114 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
device = torch.device("cuda")
set_random_seed(7)
head_dim = nope_dim + rope_dim
return dict(
q=torch.randn(3, 3, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
kv=torch.randn(5, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
indices=torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device),
indptr=torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device),
scale=head_dim**-0.5,
attn_sink=torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device),
nope_head_dim=nope_dim,
rope_head_dim=rope_dim,
)
# 448+64 is DeepSeek V4/V4.1, 512+64 is V3.2, and 256+0 stands in for the
# NoPE-only layouts where the destination spans the whole head dim.
SPARSE_PREFILL_DIMS = [(NOPE_HEAD_DIM, ROPE_HEAD_DIM), (512, 64), (256, 0)]
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_matches_allocated(
nope_dim: int, rope_dim: int
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
allocated = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)
q = kwargs["q"]
dest = torch.empty(q.shape[0], q.shape[1], nope_dim, dtype=q.dtype, device=q.device)
returned = _rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
assert returned is dest
torch.testing.assert_close(dest, allocated[..., :nope_dim], atol=0.0, rtol=0.0)
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_view_leaves_neighbors_untouched(
nope_dim: int, rope_dim: int
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
q = kwargs["q"]
num_queries, num_heads = q.shape[0], q.shape[1]
buffer = torch.full(
(num_queries + 4, num_heads + 2, nope_dim),
-7.0,
dtype=q.dtype,
device=q.device,
)
dest = buffer[2 : 2 + num_queries, :num_heads]
assert not dest.is_contiguous()
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :nope_dim]
torch.testing.assert_close(dest, expected, atol=0.0, rtol=0.0)
untouched = torch.ones_like(buffer, dtype=torch.bool)
untouched[2 : 2 + num_queries, :num_heads] = False
assert torch.all(buffer[untouched] == -7.0)
@pytest.mark.parametrize("out_dtype", [torch.float32, torch.float16])
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_casts_to_dest_dtype(
out_dtype: torch.dtype,
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(512, 64)
q = kwargs["q"]
dest = torch.empty(q.shape[0], q.shape[1], 512, dtype=out_dtype, device=q.device)
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :512]
assert dest.dtype == out_dtype
torch.testing.assert_close(dest.float(), expected.float(), atol=2e-2, rtol=2e-2)
@torch.inference_mode()
def test_sparse_attn_prefill_head_dim_wide_out_keeps_rope_rows() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(NOPE_HEAD_DIM, ROPE_HEAD_DIM)
q = kwargs["q"]
dest = torch.empty_like(q)
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _ref_sparse_prefill_ragged(
q, kwargs["kv"], [[0, 2], [1, 3, 4], []], kwargs["scale"], kwargs["attn_sink"]
)
torch.testing.assert_close(dest, expected, atol=2e-2, rtol=2e-2)
@pytest.mark.parametrize(
("num_queries", "on_gfx950", "expected"),
[(1023, True, False), (1024, True, True), (1024, False, False)],
@@ -718,10 +826,11 @@ def test_sparse_attn_prefill_preserves_dense_triton_fallback(monkeypatch) -> Non
output = torch.empty_like(q)
dense_fallback_calls = 0
def fake_dense_fallback(*args, **kwargs):
def fake_dense_fallback(*args, out, **kwargs):
nonlocal dense_fallback_calls
dense_fallback_calls += 1
return torch.zeros_like(q)
out.zero_()
return out
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: None)