[ROCm][Perf] Reach the fused QSA pre-indexer from the AMD path (#57947)

Signed-off-by: mjkvaak-amd <mjkvaak-amd@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Mikko Tukiainen
2026-10-05 16:22:29 +00:00
committed by GitHub
co-authored by Cursor
parent 528772a4bf
commit d23af12cb1
5 changed files with 1136 additions and 14 deletions
+150
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
import math
from types import SimpleNamespace
from typing import Any
@@ -9,15 +10,21 @@ import pytest
import torch
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.models.qwen4_exp.amd import indexer_qsa
from vllm.models.qwen4_exp.amd import (
model as _qwen4_exp_model, # noqa: F401
)
from vllm.models.qwen4_exp.amd import ple_layer as ple_layer_module
from vllm.models.qwen4_exp.amd.indexer_qsa import (
QSAIndexer,
apply_qsa_rmsnorm,
apply_qsa_rope,
)
from vllm.models.qwen4_exp.amd.ops import qsa as qsa_ops
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
qsa_pre_indexer,
supports_fused_pre_indexer,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
@@ -120,6 +127,149 @@ def test_qsa_rmsnorm_uses_portable_implementation(default_vllm_config) -> None:
torch.testing.assert_close(output, norm.forward_native(tensor))
def _make_indexer(*, use_fused: bool, num_tokens: int = 2) -> QSAIndexer:
"""A QSAIndexer whose unfused path runs for real, with no projection weights.
Both branches have to be reachable: a fused-path test is only meaningful if
the unfused one it replaces would otherwise have executed.
"""
heads, head_dim = 2, 4
indexer = QSAIndexer.__new__(QSAIndexer)
torch.nn.Module.__init__(indexer)
indexer.index_n_heads = heads
indexer.index_kv_heads = 1
indexer.index_head_dim = head_dim
indexer.compress_ratio = 2
indexer.use_fused_pre_indexer = use_fused
indexer.skip_topk = True
indexer.index_qk_proj = lambda hidden_states: (
torch.zeros(hidden_states.shape[0], (heads + 1) * head_dim),
None,
)
indexer.q_layernorm = GemmaRMSNorm(head_dim, eps=1e-6)
indexer.k_layernorm = GemmaRMSNorm(head_dim, eps=1e-6)
indexer.rotary_emb = SimpleNamespace(
cos_sin_cache=torch.zeros(16, head_dim),
rotary_dim=head_dim // 2,
apply_rotary_emb=lambda tensor, cos, sin: tensor,
_match_cos_sin_cache_dtype=lambda _: torch.zeros(16, head_dim),
)
# The raw cache's full width carries the MRoPE position tail that the fused
# kernel writes; key_cache is only the leading key columns.
indexer.raw_key_cache = SimpleNamespace(
kv_cache=torch.zeros(1, 4, 1, head_dim + 12),
key_cache=torch.zeros(1, 4, 1, head_dim),
cache_rope_positions=True,
rope_position_offset=head_dim,
)
indexer.compressed_key_cache = SimpleNamespace(
kv_cache=torch.zeros(1, 2, 1, head_dim)
)
raw_metadata = SimpleNamespace(
num_actual_tokens=num_tokens,
slot_mapping=torch.arange(num_tokens),
block_table=torch.zeros(1, 1, dtype=torch.int32),
query_start_loc=torch.tensor([0, num_tokens], dtype=torch.int32),
logical_positions=torch.arange(num_tokens),
token_to_req=torch.zeros(num_tokens, dtype=torch.int32),
)
compressed_metadata = SimpleNamespace(
num_actual_tokens=num_tokens,
slot_mapping=torch.arange(num_tokens),
k_work_metadata=torch.zeros(1, 2, dtype=torch.int32),
)
indexer._metadata = lambda: (raw_metadata, compressed_metadata)
indexer.test_metadata = (raw_metadata, compressed_metadata)
return indexer
@pytest.mark.usefixtures("default_vllm_config")
def test_qsa_fused_pre_indexer_replaces_the_unfused_chain(
monkeypatch: pytest.MonkeyPatch,
) -> None:
indexer = _make_indexer(use_fused=True)
calls = []
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
lambda *args, **kwargs: calls.append((args, kwargs)),
)
monkeypatch.setattr(
QSAIndexer,
"_update_and_compress",
lambda *args, **kwargs: pytest.fail("fused path ran the unfused chain"),
)
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
assert actual is rows
assert len(calls) == 1
args, kwargs = calls[0]
passed = inspect.signature(qsa_pre_indexer).bind(*args, **kwargs).arguments
_, compressed_metadata = indexer.test_metadata
# The kernel addresses the raw cache at its full width and writes the MRoPE
# position tail itself, so it needs kv_cache, not the key_cache view.
assert passed["state_cache"] is indexer.raw_key_cache.kv_cache
assert passed["compressed_cache"] is indexer.compressed_key_cache.kv_cache
assert passed["k_work_metadata"] is compressed_metadata.k_work_metadata
assert passed["compress_ratio"] == indexer.compress_ratio
assert passed["rope_pos_offset"] == indexer.index_head_dim
@pytest.mark.usefixtures("default_vllm_config")
def test_qsa_unsupported_config_keeps_the_unfused_chain(
monkeypatch: pytest.MonkeyPatch,
) -> None:
indexer = _make_indexer(use_fused=False)
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
lambda *args, **kwargs: pytest.fail("unfused path ran the fused kernel"),
)
updates = []
monkeypatch.setattr(
QSAIndexer,
"_update_and_compress",
lambda *args, **kwargs: updates.append(args),
)
rows = torch.tensor([[3, 1, -1], [5, 2, 0]], dtype=torch.int32)
actual = QSAIndexer.forward(indexer, torch.zeros(2, 8), torch.tensor([7, 8]), rows)
assert actual is rows
assert len(updates) == 1
@pytest.mark.parametrize(
("overrides", "expected"),
[
pytest.param({}, True, id="qwen3_8_flash_next"),
pytest.param({"is_neox_style": False}, False, id="not-neox"),
pytest.param({"rotary_dim": 128}, False, id="rotary-dim"),
pytest.param({"mrope_interleaved": False}, False, id="mrope-not-interleaved"),
pytest.param({"mrope_section": (11, 11, 10, 0)}, False, id="mrope-section-len"),
],
)
def test_fused_pre_indexer_gate_is_config_shaped(overrides, expected) -> None:
# Qwen3.8-Flash-Next's indexer geometry, which both vendors satisfy.
rotary = SimpleNamespace(
rotary_dim=64,
is_neox_style=True,
mrope_section=(11, 11, 10),
mrope_interleaved=True,
)
for name, value in overrides.items():
setattr(rotary, name, value)
assert (
supports_fused_pre_indexer(
rotary, head_dim=128, num_kv_heads=1, compress_ratio=4
)
is expected
)
def test_qsa_selection_uses_portable_topk_on_rocm(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -0,0 +1,344 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the AMD fused QSA pre-indexer."""
import pytest
import torch
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.models.qwen4_exp.amd.indexer_qsa import apply_qsa_rope
from vllm.models.qwen4_exp.amd.ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_store_cache_rows,
)
from vllm.models.qwen4_exp.amd.ops.qsa_pre_indexer import (
qsa_pre_indexer,
)
from vllm.models.qwen4_exp.common.qsa_cache import (
canonical_qsa_rope_positions,
circular_qsa_slot_mapping,
compressed_qsa_slot_mapping,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_rocm() or not HAS_TRITON,
reason="the AMD QSA kernels require ROCm and Triton",
)
def _reference_gemma_rmsnorm(
x: torch.Tensor,
weight: torch.Tensor,
eps: float,
) -> torch.Tensor:
"""The unfused Gemma RMSNorm the AMD reference path actually runs."""
norm = GemmaRMSNorm(weight.shape[-1], eps=eps).to(
device=weight.device, dtype=weight.dtype
)
with torch.no_grad():
norm.weight.copy_(weight)
return norm.forward_native(x)
HQ, D = 4, 128
CR = 4
MROPE_SECTION = (11, 11, 10)
EPS = 1e-6
BLOCK_SIZE = 16
COMP_PAGE = BLOCK_SIZE // CR
ROPE_POS_OFFSET = D
RTOL = 1.6e-2
ATOL = 1e-2
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
def _make_block_table(block_counts):
num_blocks = sum(block_counts)
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
physical_blocks = torch.randperm(num_blocks)
offset = 0
for request, count in enumerate(block_counts):
table[request, :count] = physical_blocks[offset : offset + count]
offset += count
return table, num_blocks
def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> None:
# e4m3 is sign-magnitude, so within a sign the uint8 code order matches the
# value order and one ulp is one code step. The two paths' intermediates
# differ in the pooling accumulation order, which at denormal magnitudes
# (absolute grid step 2^-9) shows up as up to 2 code steps.
code_diff = (
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
).abs()
abs_diff = (actual.float() - expected.float()).abs()
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize(
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
[
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
pytest.param(
True,
False,
False,
8,
*MIXED_BATCH,
id="mrope-model-1d-no-position-cache",
),
pytest.param(
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
),
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
],
)
def test_qsa_fused_pre_indexer_matches_unfused(
indexer_dtype,
mrope,
is_2d_positions,
cache_rope_positions,
state_size,
seq_lens,
query_lens,
history_lens,
) -> None:
from vllm.model_executor.layers.rotary_embedding import get_rope
if indexer_dtype == torch.float8_e4m3fn:
pytest.skip("the AMD QSA indexer cache is bf16-only")
# The FP8 comparison below is a one-ulp bound on data this test generates,
# so an unseeded run decides for itself whether it reproduces a rounding
# difference. Seed it, or the kernel's bit-exactness is only sampled.
torch.manual_seed(0)
device = "cuda"
rope_params = {
"partial_rotary_factor": 0.25,
"rope_theta": 10000000,
"rope_type": "default",
}
if mrope:
rope_params["mrope_interleaved"] = True
rope_params["mrope_section"] = list(MROPE_SECTION)
with torch.device(device):
rope = get_rope(
head_size=256,
max_position=32768,
rope_parameters=rope_params,
dtype=torch.bfloat16,
)
token_to_req = torch.cat(
[
torch.full((length,), request, dtype=torch.int32)
for request, length in enumerate(query_lens)
]
).to(device)
logical_positions = torch.cat(
[
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
for seq_len, query_len in zip(seq_lens, query_lens)
]
).to(device)
num_tokens = logical_positions.numel()
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
).to(device)
positions = (
torch.stack(
[
logical_positions,
logical_positions // 7 + 3,
logical_positions // 13 + 11,
]
)
if is_2d_positions
else logical_positions
)
position_rows = (
canonical_qsa_rope_positions(positions)
if cache_rope_positions
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
)
compressed_block_counts = [
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
]
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
compressed_block_table, num_compressed_blocks = _make_block_table(
compressed_block_counts
)
raw_block_table = raw_block_table.to(device)
raw_slots = circular_qsa_slot_mapping(
raw_block_table,
token_to_req,
logical_positions,
state_size,
query_start_loc,
)
compressed_slots = compressed_qsa_slot_mapping(
compressed_block_table.to(device),
token_to_req,
logical_positions,
COMP_PAGE,
CR,
)
group_counts = torch.tensor(
[
seq_len // CR - (seq_len - query_len) // CR
for seq_len, query_len in zip(seq_lens, query_lens)
],
dtype=torch.int32,
device=device,
)
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
work_requests = torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32, device=device),
k_work_counts,
)
local_work = torch.arange(
int(k_start_loc[-1]), dtype=torch.int32, device=device
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
k_work_metadata[: work_requests.numel()] = torch.stack(
(work_requests, local_work), dim=1
)
raw_width = D + 12 if cache_rope_positions else D
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
# blocks have a larger stride than their logical contents.
raw_page_elements = state_size * raw_width
fused_raw_storage = torch.zeros(
num_raw_blocks,
raw_page_elements + 16,
dtype=torch.bfloat16,
device=device,
)
fused_raw = torch.as_strided(
fused_raw_storage,
(num_raw_blocks, state_size, 1, raw_width),
(raw_page_elements + 16, raw_width, raw_width, 1),
)
for request, history_len in enumerate(history_lens):
history_end = seq_lens[request] - query_lens[request]
for position in range(history_end - history_len, history_end):
block = int(raw_block_table[request, 0])
row = fused_raw[block, position % state_size, 0]
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
if cache_rope_positions:
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
torch.tensor(
[position, position // 7 + 3, position // 13 + 11],
dtype=torch.int64,
device=device,
)
)
unfused_raw = fused_raw.clone()
compressed_page_elements = COMP_PAGE * D
fused_compressed_storage = torch.zeros(
num_compressed_blocks,
compressed_page_elements + 16,
dtype=indexer_dtype,
device=device,
)
fused_compressed = torch.as_strided(
fused_compressed_storage,
(num_compressed_blocks, COMP_PAGE, 1, D),
(compressed_page_elements + 16, D, D, 1),
)
unfused_compressed = fused_compressed.clone()
projected_qk = torch.randn(
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
)
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
qsa_pre_indexer(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
positions,
rope.cos_sin_cache,
q_weight,
k_weight,
EPS,
fused_query,
fused_raw,
raw_slots,
raw_block_table,
query_start_loc,
logical_positions,
fused_compressed,
compressed_slots,
k_work_metadata,
compress_ratio=CR,
mrope_section=MROPE_SECTION if mrope else None,
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
)
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
unfused_query = _reference_gemma_rmsnorm(
unfused_query.reshape(-1, D), q_weight, EPS
).reshape_as(unfused_query)
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
raw_keys = unfused_raw[..., :D]
rope_positions = (
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
if cache_rope_positions
else None
)
pooled, first_positions = qsa_compress_groups_with_ratio(
projected_qk[:, HQ * D :].reshape(-1, 1, D),
position_rows,
raw_keys,
raw_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
CR,
rope_positions,
)
compressed_rows = _reference_gemma_rmsnorm(
pooled.reshape(-1, D), k_weight, EPS
).reshape(-1, 1, D)
group_positions = (
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
)
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
if rope_positions is not None:
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
if indexer_dtype == torch.float8_e4m3fn:
# Both paths round the same ~bf16 intermediates to e4m3. Bitwise
# equality (the dsv4 indexer test's bar) does not hold here: the 1D
# reference rope (forward_cuda) differs from _norm_rope by up to 1
# bf16 ulp, and even the MRoPE pair flips a code occasionally.
unfused_query = unfused_query.to(indexer_dtype)
assert_fp8_within_one_ulp(fused_query, unfused_query)
else:
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
if indexer_dtype == torch.float8_e4m3fn:
assert_fp8_within_one_ulp(fused_compressed, unfused_compressed)
else:
torch.testing.assert_close(
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
)