[ROCm][Perf] Fuse main QK-norm/RoPE/gate and KV-cache write into the AMD QSA prepare launch (#59732)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Mikko Tukiainen
2026-10-09 21:14:49 +08:00
committed by GitHub
co-authored by Cursor
parent 9adba10f3f
commit 81c993658e
7 changed files with 454 additions and 68 deletions
+58 -12
View File
@@ -21,8 +21,8 @@ from vllm.models.qwen4_exp.amd.indexer_qsa import (
apply_qsa_rope,
)
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
qsa_pre_indexer,
from vllm.models.qwen4_exp.amd.ops.qsa_prepare import (
qsa_prepare,
supports_fused_pre_indexer,
)
from vllm.platforms import current_platform
@@ -127,6 +127,18 @@ def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
torch.testing.assert_close(output, norm.forward_native(tensor))
def _make_attn(*, use_fused: bool) -> SimpleNamespace:
"""The main-attention fields the fused QSA prepare reads from its owner."""
return SimpleNamespace(
use_fused_qsa_prepare=use_fused,
kv_cache=torch.zeros(1, 2, 16, 8),
q_norm=GemmaRMSNorm(4, eps=1e-6),
k_norm=GemmaRMSNorm(4, eps=1e-6),
_k_scale_float=1.0,
_v_scale_float=1.0,
)
def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer:
"""A QSAIndexer whose unfused path runs for real, with no projection weights.
@@ -184,29 +196,45 @@ def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer:
@pytest.mark.usefixtures("default_vllm_config")
def test_qsa_fused_pre_indexer_replaces_the_unfused_chain(
def test_qsa_fused_prepare_replaces_the_unfused_chain(
monkeypatch: pytest.MonkeyPatch,
) -> None:
indexer = _make_indexer(use_fused=True)
attn = _make_attn(use_fused=True)
calls = []
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
lambda *args, **kwargs: calls.append((args, kwargs)),
)
main_outputs = (torch.empty(0), torch.empty(0))
def fake_prepare(*args, **kwargs):
calls.append((args, kwargs))
return main_outputs
monkeypatch.setattr(indexer_qsa, "qsa_prepare", fake_prepare)
monkeypatch.setattr(
QSAIndexer,
"_update_and_compress",
lambda *args, **kwargs: pytest.fail("fused path ran the unfused chain"),
)
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
qkv = torch.zeros(4, 16)
slot_mapping = torch.arange(4)
gate_out = torch.empty(4, 1, 4)
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
actual, actual_main = QSAIndexer.forward(
indexer,
torch.zeros(2, 8),
torch.tensor([7, 8]),
rows,
attn=attn,
qkv=qkv,
slot_mapping=slot_mapping,
gate_out=gate_out,
)
assert actual is rows
assert actual_main is main_outputs
assert len(calls) == 1
args, kwargs = calls[0]
passed = inspect.signature(qsa_pre_indexer).bind(*args, **kwargs).arguments
passed = inspect.signature(qsa_prepare).bind(*args, **kwargs).arguments
_, compressed_metadata = indexer.test_metadata
# The kernel addresses the raw cache at its full width and writes the MRoPE
# position tail itself, so it needs kv_cache, not the key_cache view.
@@ -215,6 +243,17 @@ def test_qsa_fused_pre_indexer_replaces_the_unfused_chain(
assert passed["k_work_metadata"] is compressed_metadata.k_work_metadata
assert passed["compress_ratio"] == indexer.compress_ratio
assert passed["rope_pos_offset"] == indexer.index_head_dim
# Main attention: only the real tokens, the vLLM cache viewed per token,
# and the gate landing in the caller's buffer.
assert passed["main_qkv"].shape[0] == 2
assert passed["main_qkv"].data_ptr() == qkv.data_ptr()
assert passed["main_slot_mapping"].shape == (2,)
assert passed["main_kv_cache"].shape == (1, 16, 2, 8)
assert passed["main_kv_cache"].data_ptr() == attn.kv_cache.data_ptr()
assert passed["main_gate_out"].shape[0] == 2
assert passed["main_gate_out"].data_ptr() == gate_out.data_ptr()
assert passed["main_q_norm_weight"] is attn.q_norm.weight
assert passed["main_k_norm_weight"] is attn.k_norm.weight
@pytest.mark.usefixtures("default_vllm_config")
@@ -224,7 +263,7 @@ def test_qsa_unsupported_config_keeps_the_unfused_chain(
indexer = _make_indexer(use_fused=False)
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
"qsa_prepare",
lambda *args, **kwargs: pytest.fail("unfused path ran the fused kernel"),
)
updates = []
@@ -235,9 +274,16 @@ def test_qsa_unsupported_config_keeps_the_unfused_chain(
)
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
actual, main_outputs = QSAIndexer.forward(
indexer,
torch.zeros(2, 8),
torch.tensor([7, 8]),
rows,
attn=_make_attn(use_fused=False),
)
assert actual is rows
assert main_outputs is None
assert len(updates) == 1
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the AMD fused QSA pre-indexer."""
"""Correctness tests for the AMD fused QSA prepare kernel."""
import pytest
import torch
@@ -11,9 +11,7 @@ from vllm.models.qwen4_exp.amd.ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_store_cache_rows,
)
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
qsa_pre_indexer,
)
from vllm.models.qwen4_exp.amd.ops.qsa_prepare import qsa_prepare
from vllm.models.qwen4_exp.common.qsa_cache import (
canonical_qsa_rope_positions,
circular_qsa_slot_mapping,
@@ -52,6 +50,7 @@ ROPE_POS_OFFSET = D
RTOL = 1.6e-2
ATOL = 1e-2
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16
def _make_block_table(block_counts):
@@ -78,6 +77,76 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
def _make_main_inputs(num_tokens: int) -> dict:
"""Random main-attention arguments with a paged BF16 cache in vLLM's layout."""
num_blocks = num_tokens // MAIN_PAGE + 2
cache = torch.zeros(
num_blocks,
MAIN_HK,
MAIN_PAGE,
2 * MAIN_D,
dtype=torch.bfloat16,
device="cuda",
).transpose(1, 2)
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
slots[-1] = -1
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
return dict(
main_qkv=torch.randn(
num_tokens,
2 * (MAIN_HQ + MAIN_HK) * MAIN_D,
dtype=torch.bfloat16,
device="cuda",
),
main_q_norm_weight=norm_weights[0],
main_k_norm_weight=norm_weights[1],
main_eps=EPS,
main_kv_cache=cache,
main_slot_mapping=slots,
main_k_scale=1.0,
main_v_scale=1.0,
# The AMD op hands the gate back through a buffer it does not own.
main_gate_out=torch.empty(
num_tokens, MAIN_HQ, MAIN_D, dtype=torch.bfloat16, device="cuda"
),
)
def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None:
"""Compare with the AMD layer's unfused split, QK-norm, RoPE and cache write."""
from vllm._custom_ops import reshape_and_cache_flash
num_tokens = main["main_qkv"].shape[0]
q_gate, k, v = main["main_qkv"].split(
[2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1
)
q, gate = torch.chunk(q_gate.view(num_tokens, MAIN_HQ, -1), 2, dim=-1)
q = _reference_gemma_rmsnorm(
q.reshape(-1, MAIN_D), main["main_q_norm_weight"], EPS
).view(num_tokens, -1)
k = _reference_gemma_rmsnorm(
k.reshape(-1, MAIN_D), main["main_k_norm_weight"], EPS
).view(num_tokens, -1)
q, k = rope(positions, q, k)
kv_cache = main["main_kv_cache"]
cache = torch.zeros_like(kv_cache)
key_cache, value_cache = cache.split(MAIN_D, dim=-1)
scale = torch.tensor(1.0, device="cuda")
reshape_and_cache_flash(
k.view(-1, MAIN_HK, MAIN_D),
v.view(-1, MAIN_HK, MAIN_D),
key_cache,
value_cache,
main["main_slot_mapping"],
"auto",
scale,
scale,
)
torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL)
assert torch.equal(gate_out, gate)
torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL)
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
@@ -103,7 +172,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
],
)
def test_qsa_fused_pre_indexer_matches_unfused(
def test_qsa_fused_prepare_matches_unfused(
indexer_dtype,
mrope,
is_2d_positions,
@@ -269,7 +338,8 @@ def test_qsa_fused_pre_indexer_matches_unfused(
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
qsa_pre_indexer(
main = _make_main_inputs(num_tokens)
main_q_out, main_gate_out = qsa_prepare(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
positions,
@@ -289,7 +359,10 @@ def test_qsa_fused_pre_indexer_matches_unfused(
compress_ratio=CR,
mrope_section=MROPE_SECTION if mrope else None,
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
**main,
)
assert main_gate_out is main["main_gate_out"]
_check_main_outputs(main, main_q_out, main_gate_out, rope, positions)
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
unfused_query = _reference_gemma_rmsnorm(