[ROCm][Perf] Fuse main QK-norm/RoPE/gate and KV-cache write into the AMD QSA prepare launch (#59732)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
9adba10f3f
commit
81c993658e
@@ -21,8 +21,8 @@ from vllm.models.qwen4_exp.amd.indexer_qsa import (
|
||||
apply_qsa_rope,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_prepare import (
|
||||
qsa_prepare,
|
||||
supports_fused_pre_indexer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -127,6 +127,18 @@ def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
|
||||
torch.testing.assert_close(output, norm.forward_native(tensor))
|
||||
|
||||
|
||||
def _make_attn(*, use_fused: bool) -> SimpleNamespace:
|
||||
"""The main-attention fields the fused QSA prepare reads from its owner."""
|
||||
return SimpleNamespace(
|
||||
use_fused_qsa_prepare=use_fused,
|
||||
kv_cache=torch.zeros(1, 2, 16, 8),
|
||||
q_norm=GemmaRMSNorm(4, eps=1e-6),
|
||||
k_norm=GemmaRMSNorm(4, eps=1e-6),
|
||||
_k_scale_float=1.0,
|
||||
_v_scale_float=1.0,
|
||||
)
|
||||
|
||||
|
||||
def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer:
|
||||
"""A QSAIndexer whose unfused path runs for real, with no projection weights.
|
||||
|
||||
@@ -184,29 +196,45 @@ def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer:
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_qsa_fused_pre_indexer_replaces_the_unfused_chain(
|
||||
def test_qsa_fused_prepare_replaces_the_unfused_chain(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
indexer = _make_indexer(use_fused=True)
|
||||
attn = _make_attn(use_fused=True)
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
lambda *args, **kwargs: calls.append((args, kwargs)),
|
||||
)
|
||||
main_outputs = (torch.empty(0), torch.empty(0))
|
||||
|
||||
def fake_prepare(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
return main_outputs
|
||||
|
||||
monkeypatch.setattr(indexer_qsa, "qsa_prepare", fake_prepare)
|
||||
monkeypatch.setattr(
|
||||
QSAIndexer,
|
||||
"_update_and_compress",
|
||||
lambda *args, **kwargs: pytest.fail("fused path ran the unfused chain"),
|
||||
)
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
qkv = torch.zeros(4, 16)
|
||||
slot_mapping = torch.arange(4)
|
||||
gate_out = torch.empty(4, 1, 4)
|
||||
|
||||
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
|
||||
actual, actual_main = QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 8),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
attn=attn,
|
||||
qkv=qkv,
|
||||
slot_mapping=slot_mapping,
|
||||
gate_out=gate_out,
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
assert actual_main is main_outputs
|
||||
assert len(calls) == 1
|
||||
args, kwargs = calls[0]
|
||||
passed = inspect.signature(qsa_pre_indexer).bind(*args, **kwargs).arguments
|
||||
passed = inspect.signature(qsa_prepare).bind(*args, **kwargs).arguments
|
||||
_, compressed_metadata = indexer.test_metadata
|
||||
# The kernel addresses the raw cache at its full width and writes the MRoPE
|
||||
# position tail itself, so it needs kv_cache, not the key_cache view.
|
||||
@@ -215,6 +243,17 @@ def test_qsa_fused_pre_indexer_replaces_the_unfused_chain(
|
||||
assert passed["k_work_metadata"] is compressed_metadata.k_work_metadata
|
||||
assert passed["compress_ratio"] == indexer.compress_ratio
|
||||
assert passed["rope_pos_offset"] == indexer.index_head_dim
|
||||
# Main attention: only the real tokens, the vLLM cache viewed per token,
|
||||
# and the gate landing in the caller's buffer.
|
||||
assert passed["main_qkv"].shape[0] == 2
|
||||
assert passed["main_qkv"].data_ptr() == qkv.data_ptr()
|
||||
assert passed["main_slot_mapping"].shape == (2,)
|
||||
assert passed["main_kv_cache"].shape == (1, 16, 2, 8)
|
||||
assert passed["main_kv_cache"].data_ptr() == attn.kv_cache.data_ptr()
|
||||
assert passed["main_gate_out"].shape[0] == 2
|
||||
assert passed["main_gate_out"].data_ptr() == gate_out.data_ptr()
|
||||
assert passed["main_q_norm_weight"] is attn.q_norm.weight
|
||||
assert passed["main_k_norm_weight"] is attn.k_norm.weight
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@@ -224,7 +263,7 @@ def test_qsa_unsupported_config_keeps_the_unfused_chain(
|
||||
indexer = _make_indexer(use_fused=False)
|
||||
monkeypatch.setattr(
|
||||
indexer_qsa,
|
||||
"qsa_pre_indexer",
|
||||
"qsa_prepare",
|
||||
lambda *args, **kwargs: pytest.fail("unfused path ran the fused kernel"),
|
||||
)
|
||||
updates = []
|
||||
@@ -235,9 +274,16 @@ def test_qsa_unsupported_config_keeps_the_unfused_chain(
|
||||
)
|
||||
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
|
||||
|
||||
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
|
||||
actual, main_outputs = QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 8),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
attn=_make_attn(use_fused=False),
|
||||
)
|
||||
|
||||
assert actual is rows
|
||||
assert main_outputs is None
|
||||
assert len(updates) == 1
|
||||
|
||||
|
||||
|
||||
+79
-6
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the AMD fused QSA pre-indexer."""
|
||||
"""Correctness tests for the AMD fused QSA prepare kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -11,9 +11,7 @@ from vllm.models.qwen4_exp.amd.ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
|
||||
qsa_pre_indexer,
|
||||
)
|
||||
from vllm.models.qwen4_exp.amd.ops.qsa_prepare import qsa_prepare
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||
canonical_qsa_rope_positions,
|
||||
circular_qsa_slot_mapping,
|
||||
@@ -52,6 +50,7 @@ ROPE_POS_OFFSET = D
|
||||
RTOL = 1.6e-2
|
||||
ATOL = 1e-2
|
||||
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
|
||||
MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16
|
||||
|
||||
|
||||
def _make_block_table(block_counts):
|
||||
@@ -78,6 +77,76 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
|
||||
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
|
||||
|
||||
|
||||
def _make_main_inputs(num_tokens: int) -> dict:
|
||||
"""Random main-attention arguments with a paged BF16 cache in vLLM's layout."""
|
||||
num_blocks = num_tokens // MAIN_PAGE + 2
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
MAIN_HK,
|
||||
MAIN_PAGE,
|
||||
2 * MAIN_D,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
).transpose(1, 2)
|
||||
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
|
||||
slots[-1] = -1
|
||||
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
|
||||
return dict(
|
||||
main_qkv=torch.randn(
|
||||
num_tokens,
|
||||
2 * (MAIN_HQ + MAIN_HK) * MAIN_D,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
),
|
||||
main_q_norm_weight=norm_weights[0],
|
||||
main_k_norm_weight=norm_weights[1],
|
||||
main_eps=EPS,
|
||||
main_kv_cache=cache,
|
||||
main_slot_mapping=slots,
|
||||
main_k_scale=1.0,
|
||||
main_v_scale=1.0,
|
||||
# The AMD op hands the gate back through a buffer it does not own.
|
||||
main_gate_out=torch.empty(
|
||||
num_tokens, MAIN_HQ, MAIN_D, dtype=torch.bfloat16, device="cuda"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None:
|
||||
"""Compare with the AMD layer's unfused split, QK-norm, RoPE and cache write."""
|
||||
from vllm._custom_ops import reshape_and_cache_flash
|
||||
|
||||
num_tokens = main["main_qkv"].shape[0]
|
||||
q_gate, k, v = main["main_qkv"].split(
|
||||
[2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1
|
||||
)
|
||||
q, gate = torch.chunk(q_gate.view(num_tokens, MAIN_HQ, -1), 2, dim=-1)
|
||||
q = _reference_gemma_rmsnorm(
|
||||
q.reshape(-1, MAIN_D), main["main_q_norm_weight"], EPS
|
||||
).view(num_tokens, -1)
|
||||
k = _reference_gemma_rmsnorm(
|
||||
k.reshape(-1, MAIN_D), main["main_k_norm_weight"], EPS
|
||||
).view(num_tokens, -1)
|
||||
q, k = rope(positions, q, k)
|
||||
kv_cache = main["main_kv_cache"]
|
||||
cache = torch.zeros_like(kv_cache)
|
||||
key_cache, value_cache = cache.split(MAIN_D, dim=-1)
|
||||
scale = torch.tensor(1.0, device="cuda")
|
||||
reshape_and_cache_flash(
|
||||
k.view(-1, MAIN_HK, MAIN_D),
|
||||
v.view(-1, MAIN_HK, MAIN_D),
|
||||
key_cache,
|
||||
value_cache,
|
||||
main["main_slot_mapping"],
|
||||
"auto",
|
||||
scale,
|
||||
scale,
|
||||
)
|
||||
torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(gate_out, gate)
|
||||
torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@@ -103,7 +172,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
|
||||
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
def test_qsa_fused_prepare_matches_unfused(
|
||||
indexer_dtype,
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
@@ -269,7 +338,8 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
|
||||
qsa_pre_indexer(
|
||||
main = _make_main_inputs(num_tokens)
|
||||
main_q_out, main_gate_out = qsa_prepare(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
positions,
|
||||
@@ -289,7 +359,10 @@ def test_qsa_fused_pre_indexer_matches_unfused(
|
||||
compress_ratio=CR,
|
||||
mrope_section=MROPE_SECTION if mrope else None,
|
||||
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
|
||||
**main,
|
||||
)
|
||||
assert main_gate_out is main["main_gate_out"]
|
||||
_check_main_outputs(main, main_q_out, main_gate_out, rope, positions)
|
||||
|
||||
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
|
||||
unfused_query = _reference_gemma_rmsnorm(
|
||||
Reference in New Issue
Block a user