[Model][Hardware][AMD][Kernel] Part 2/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_ATTN (#42755)

Signed-off-by: Jack Hu <Jack.Hu@amd.com>
Signed-off-by: Jack Hu <jingxujaster@hotmail.com>
Signed-off-by: Jack Hu <jack.hu@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Jack Hu
2026-10-15 17:10:14 -07:00
committed by GitHub
co-authored by Douglas Lehr
parent af28c171f5
commit ef38e8d006
23 changed files with 701 additions and 205 deletions
+25 -1
View File
@@ -122,6 +122,7 @@ def ref_single_query_cached_kv_attention(
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("use_interleaved_v_cache", [False, True])
def test_paged_attention(
kv_cache_factory,
num_seqs: int,
@@ -133,9 +134,14 @@ def test_paged_attention(
kv_cache_dtype: str,
seed: int,
device: str,
use_interleaved_v_cache: bool,
) -> None:
if current_platform.is_navi() and (
kv_cache_dtype == "fp8" or head_size != 128 or block_size != 16 or use_alibi
kv_cache_dtype == "fp8"
or head_size != 128
or block_size != 16
or use_alibi
or use_interleaved_v_cache
):
pytest.skip()
@@ -201,6 +207,20 @@ def test_paged_attention(
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
standard_value_cache = value_cache
if use_interleaved_v_cache:
# Hand the kernel V in the interleaved layout (same 4D shape); the
# reference below gets the standard layout back.
x_v = 16 // value_cache.element_size()
value_cache = (
value_cache.view(
NUM_BLOCKS, num_kv_heads, head_size, block_size // x_v, x_v
)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(NUM_BLOCKS, num_kv_heads, head_size, block_size)
)
# Call the paged attention kernel.
output = torch.empty_like(query)
num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
@@ -234,6 +254,7 @@ def test_paged_attention(
kv_cache_dtype,
k_scale,
v_scale,
use_interleaved_v_cache=use_interleaved_v_cache,
)
opcheck(
@@ -259,10 +280,13 @@ def test_paged_attention(
v_scale,
None,
"f16",
use_interleaved_v_cache,
),
cond=(head_size == 64 and block_size == BLOCK_SIZES[0]),
)
value_cache = standard_value_cache
# Run the reference implementation.
if kv_cache_dtype == "fp8":
# Convert cache data back to dtype.
+90 -2
View File
@@ -109,6 +109,7 @@ def create_alibi_causal_mask(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention(
num_heads: int,
@@ -119,6 +120,7 @@ def test_contexted_kv_attention(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
block_size: int = 32,
) -> None:
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
@@ -225,6 +227,23 @@ def test_contexted_kv_attention(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
extra_op_kwargs = {}
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
# read it back interleaved.
x_v = 16 // v_cache.element_size()
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
if op is context_attention_fwd:
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
else:
extra_op_kwargs["use_interleaved_v_cache"] = True
# Warm up the Triton kernel by calling it once before actually measuring
# generation time
op(
@@ -243,6 +262,7 @@ def test_contexted_kv_attention(
k_scale,
v_scale,
sliding_window=sliding_window,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
start_time = time.time()
@@ -262,6 +282,7 @@ def test_contexted_kv_attention(
k_scale,
v_scale,
sliding_window=sliding_window,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
end_time = time.time()
@@ -386,6 +407,7 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_cached_kv(
num_heads: int,
@@ -394,6 +416,7 @@ def test_contexted_kv_attention_cached_kv(
dtype: torch.dtype,
kv_cache_dtype: str,
device: str,
interleave_v: bool,
block_size: int = 32,
) -> None:
# Exercises the KV_FROM_CACHE path of context_attention_fwd: the current
@@ -487,6 +510,18 @@ def test_contexted_kv_attention_cached_kv(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
interleaved_v_pack_factor = 0
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape).
interleaved_v_pack_factor = 16 // v_cache.element_size()
x_v = interleaved_v_pack_factor
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
# Cached-K/V path: current-chunk k and v are None.
context_attention_fwd(
query,
@@ -504,6 +539,7 @@ def test_contexted_kv_attention_cached_kv(
k_scale,
v_scale,
sliding_window=0,
interleaved_v_pack_factor=interleaved_v_pack_factor,
)
torch.accelerator.synchronize()
@@ -544,8 +580,11 @@ def test_contexted_kv_attention_cached_kv(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> None:
def test_contexted_kv_attention_cached_kv_block_table_boundary(
device: str, interleave_v: bool
) -> None:
# Boundary guard for the KV_FROM_CACHE block-table load. With an
# exact-sized block table (row length == number of blocks for the
# sequence) and a query that ends the sequence, the last K/V tile has
@@ -611,6 +650,18 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
interleaved_v_pack_factor = 0
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape).
interleaved_v_pack_factor = 16 // v_cache.element_size()
x_v = interleaved_v_pack_factor
v_cache = (
v_cache.view(num_blocks, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(num_blocks, num_kv_heads, head_size, block_size)
)
# Cached-K/V path: current-chunk k and v are None.
context_attention_fwd(
query,
@@ -628,6 +679,7 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
k_scale,
v_scale,
sliding_window=0,
interleaved_v_pack_factor=interleaved_v_pack_factor,
)
torch.accelerator.synchronize()
@@ -723,6 +775,7 @@ def test_contexted_kv_attention_cached_kv_alibi_unsupported(device: str) -> None
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_alibi(
num_heads: int,
@@ -732,6 +785,7 @@ def test_contexted_kv_attention_alibi(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
block_size: int = 32,
) -> None:
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
@@ -860,6 +914,23 @@ def test_contexted_kv_attention_alibi(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
extra_op_kwargs = {}
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
# read it back interleaved.
x_v = 16 // v_cache.element_size()
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
if op is context_attention_fwd:
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
else:
extra_op_kwargs["use_interleaved_v_cache"] = True
# Warm up the Triton kernel by calling it once before actually measuring
# generation time
op(
@@ -878,6 +949,7 @@ def test_contexted_kv_attention_alibi(
k_scale,
v_scale,
alibi_slopes=alibi_slopes,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
start_time = time.time()
@@ -897,6 +969,7 @@ def test_contexted_kv_attention_alibi(
k_scale,
v_scale,
alibi_slopes=alibi_slopes,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
end_time = time.time()
@@ -992,6 +1065,7 @@ def test_contexted_kv_attention_alibi(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_f32(
num_heads: int,
@@ -1002,6 +1076,7 @@ def test_contexted_kv_attention_f32(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
test_contexted_kv_attention(
num_heads,
@@ -1012,6 +1087,7 @@ def test_contexted_kv_attention_f32(
kv_cache_dtype,
device,
op,
interleave_v=interleave_v,
)
@@ -1023,6 +1099,7 @@ def test_contexted_kv_attention_f32(
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_alibi_f32(
num_heads: int,
@@ -1032,9 +1109,17 @@ def test_contexted_kv_attention_alibi_f32(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
test_contexted_kv_attention_alibi(
num_heads, num_queries_per_kv, head_size, dtype, kv_cache_dtype, device, op
num_heads,
num_queries_per_kv,
head_size,
dtype,
kv_cache_dtype,
device,
op,
interleave_v=interleave_v,
)
@@ -1052,6 +1137,7 @@ NONSTANDARD_BLOCK_SIZE_SHAPES = [
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_qwen3_nonstandard_block_size(
num_heads: int,
@@ -1061,6 +1147,7 @@ def test_qwen3_nonstandard_block_size(
dtype: torch.dtype,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
"""Non-power-of-2 pages must match, even when a tile straddles a page."""
if not current_platform.is_rocm():
@@ -1076,4 +1163,5 @@ def test_qwen3_nonstandard_block_size(
kv_cache_dtype="auto",
device=device,
op=op,
interleave_v=interleave_v,
)
@@ -1061,3 +1061,25 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
rtol=rtol,
)
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
def test_asm_kv_scale_tables_shared_across_builders():
"""Builders share one AsmKvScaleTables; equal (shape, scale) requests must
return the same dense table, a different scale a different one."""
from vllm.v1.attention.backends.rocm_aiter_fa import (
AiterFlashAttentionMetadataBuilder,
AsmKvScaleTables,
)
tables = AsmKvScaleTables()
device = torch.device("cuda")
table = tables.get(device, 8, 2, 16, 0.5)
assert tables.get(device, 8, 2, 16, 0.5) is table
assert tables.get(device, 8, 2, 16, 1.0) is not table
assert table.shape == (8, 2, 16)
assert table.dtype == torch.float32
assert torch.all(table == 0.5)
# The builder class holds the single process-wide instance.
assert isinstance(
AiterFlashAttentionMetadataBuilder.asm_kv_scale_tables, AsmKvScaleTables
)