[Model][Hardware][AMD][Kernel] Part 2/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_ATTN (#42755)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Signed-off-by: Jack Hu <jingxujaster@hotmail.com> Signed-off-by: Jack Hu <jack.hu@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
@@ -122,6 +122,7 @@ def ref_single_query_cached_kv_attention(
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("use_interleaved_v_cache", [False, True])
|
||||
def test_paged_attention(
|
||||
kv_cache_factory,
|
||||
num_seqs: int,
|
||||
@@ -133,9 +134,14 @@ def test_paged_attention(
|
||||
kv_cache_dtype: str,
|
||||
seed: int,
|
||||
device: str,
|
||||
use_interleaved_v_cache: bool,
|
||||
) -> None:
|
||||
if current_platform.is_navi() and (
|
||||
kv_cache_dtype == "fp8" or head_size != 128 or block_size != 16 or use_alibi
|
||||
kv_cache_dtype == "fp8"
|
||||
or head_size != 128
|
||||
or block_size != 16
|
||||
or use_alibi
|
||||
or use_interleaved_v_cache
|
||||
):
|
||||
pytest.skip()
|
||||
|
||||
@@ -201,6 +207,20 @@ def test_paged_attention(
|
||||
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
|
||||
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
|
||||
|
||||
standard_value_cache = value_cache
|
||||
if use_interleaved_v_cache:
|
||||
# Hand the kernel V in the interleaved layout (same 4D shape); the
|
||||
# reference below gets the standard layout back.
|
||||
x_v = 16 // value_cache.element_size()
|
||||
value_cache = (
|
||||
value_cache.view(
|
||||
NUM_BLOCKS, num_kv_heads, head_size, block_size // x_v, x_v
|
||||
)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(NUM_BLOCKS, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Call the paged attention kernel.
|
||||
output = torch.empty_like(query)
|
||||
num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
|
||||
@@ -234,6 +254,7 @@ def test_paged_attention(
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
use_interleaved_v_cache=use_interleaved_v_cache,
|
||||
)
|
||||
|
||||
opcheck(
|
||||
@@ -259,10 +280,13 @@ def test_paged_attention(
|
||||
v_scale,
|
||||
None,
|
||||
"f16",
|
||||
use_interleaved_v_cache,
|
||||
),
|
||||
cond=(head_size == 64 and block_size == BLOCK_SIZES[0]),
|
||||
)
|
||||
|
||||
value_cache = standard_value_cache
|
||||
|
||||
# Run the reference implementation.
|
||||
if kv_cache_dtype == "fp8":
|
||||
# Convert cache data back to dtype.
|
||||
|
||||
@@ -109,6 +109,7 @@ def create_alibi_causal_mask(
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention(
|
||||
num_heads: int,
|
||||
@@ -119,6 +120,7 @@ def test_contexted_kv_attention(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
|
||||
@@ -225,6 +227,23 @@ def test_contexted_kv_attention(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
extra_op_kwargs = {}
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
|
||||
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
|
||||
# read it back interleaved.
|
||||
x_v = 16 // v_cache.element_size()
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
if op is context_attention_fwd:
|
||||
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
|
||||
else:
|
||||
extra_op_kwargs["use_interleaved_v_cache"] = True
|
||||
|
||||
# Warm up the Triton kernel by calling it once before actually measuring
|
||||
# generation time
|
||||
op(
|
||||
@@ -243,6 +262,7 @@ def test_contexted_kv_attention(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=sliding_window,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.time()
|
||||
@@ -262,6 +282,7 @@ def test_contexted_kv_attention(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=sliding_window,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
end_time = time.time()
|
||||
@@ -386,6 +407,7 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_cached_kv(
|
||||
num_heads: int,
|
||||
@@ -394,6 +416,7 @@ def test_contexted_kv_attention_cached_kv(
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
# Exercises the KV_FROM_CACHE path of context_attention_fwd: the current
|
||||
@@ -487,6 +510,18 @@ def test_contexted_kv_attention_cached_kv(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
interleaved_v_pack_factor = 0
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape).
|
||||
interleaved_v_pack_factor = 16 // v_cache.element_size()
|
||||
x_v = interleaved_v_pack_factor
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Cached-K/V path: current-chunk k and v are None.
|
||||
context_attention_fwd(
|
||||
query,
|
||||
@@ -504,6 +539,7 @@ def test_contexted_kv_attention_cached_kv(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=0,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -544,8 +580,11 @@ def test_contexted_kv_attention_cached_kv(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> None:
|
||||
def test_contexted_kv_attention_cached_kv_block_table_boundary(
|
||||
device: str, interleave_v: bool
|
||||
) -> None:
|
||||
# Boundary guard for the KV_FROM_CACHE block-table load. With an
|
||||
# exact-sized block table (row length == number of blocks for the
|
||||
# sequence) and a query that ends the sequence, the last K/V tile has
|
||||
@@ -611,6 +650,18 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
interleaved_v_pack_factor = 0
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape).
|
||||
interleaved_v_pack_factor = 16 // v_cache.element_size()
|
||||
x_v = interleaved_v_pack_factor
|
||||
v_cache = (
|
||||
v_cache.view(num_blocks, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(num_blocks, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Cached-K/V path: current-chunk k and v are None.
|
||||
context_attention_fwd(
|
||||
query,
|
||||
@@ -628,6 +679,7 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=0,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -723,6 +775,7 @@ def test_contexted_kv_attention_cached_kv_alibi_unsupported(device: str) -> None
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_alibi(
|
||||
num_heads: int,
|
||||
@@ -732,6 +785,7 @@ def test_contexted_kv_attention_alibi(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
|
||||
@@ -860,6 +914,23 @@ def test_contexted_kv_attention_alibi(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
extra_op_kwargs = {}
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
|
||||
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
|
||||
# read it back interleaved.
|
||||
x_v = 16 // v_cache.element_size()
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
if op is context_attention_fwd:
|
||||
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
|
||||
else:
|
||||
extra_op_kwargs["use_interleaved_v_cache"] = True
|
||||
|
||||
# Warm up the Triton kernel by calling it once before actually measuring
|
||||
# generation time
|
||||
op(
|
||||
@@ -878,6 +949,7 @@ def test_contexted_kv_attention_alibi(
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.time()
|
||||
@@ -897,6 +969,7 @@ def test_contexted_kv_attention_alibi(
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
end_time = time.time()
|
||||
@@ -992,6 +1065,7 @@ def test_contexted_kv_attention_alibi(
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_f32(
|
||||
num_heads: int,
|
||||
@@ -1002,6 +1076,7 @@ def test_contexted_kv_attention_f32(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
test_contexted_kv_attention(
|
||||
num_heads,
|
||||
@@ -1012,6 +1087,7 @@ def test_contexted_kv_attention_f32(
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
|
||||
@@ -1023,6 +1099,7 @@ def test_contexted_kv_attention_f32(
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_alibi_f32(
|
||||
num_heads: int,
|
||||
@@ -1032,9 +1109,17 @@ def test_contexted_kv_attention_alibi_f32(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
test_contexted_kv_attention_alibi(
|
||||
num_heads, num_queries_per_kv, head_size, dtype, kv_cache_dtype, device, op
|
||||
num_heads,
|
||||
num_queries_per_kv,
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
|
||||
@@ -1052,6 +1137,7 @@ NONSTANDARD_BLOCK_SIZE_SHAPES = [
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_qwen3_nonstandard_block_size(
|
||||
num_heads: int,
|
||||
@@ -1061,6 +1147,7 @@ def test_qwen3_nonstandard_block_size(
|
||||
dtype: torch.dtype,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
"""Non-power-of-2 pages must match, even when a tile straddles a page."""
|
||||
if not current_platform.is_rocm():
|
||||
@@ -1076,4 +1163,5 @@ def test_qwen3_nonstandard_block_size(
|
||||
kv_cache_dtype="auto",
|
||||
device=device,
|
||||
op=op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
@@ -1061,3 +1061,25 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
|
||||
rtol=rtol,
|
||||
)
|
||||
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
def test_asm_kv_scale_tables_shared_across_builders():
|
||||
"""Builders share one AsmKvScaleTables; equal (shape, scale) requests must
|
||||
return the same dense table, a different scale a different one."""
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import (
|
||||
AiterFlashAttentionMetadataBuilder,
|
||||
AsmKvScaleTables,
|
||||
)
|
||||
|
||||
tables = AsmKvScaleTables()
|
||||
device = torch.device("cuda")
|
||||
table = tables.get(device, 8, 2, 16, 0.5)
|
||||
assert tables.get(device, 8, 2, 16, 0.5) is table
|
||||
assert tables.get(device, 8, 2, 16, 1.0) is not table
|
||||
assert table.shape == (8, 2, 16)
|
||||
assert table.dtype == torch.float32
|
||||
assert torch.all(table == 0.5)
|
||||
# The builder class holds the single process-wide instance.
|
||||
assert isinstance(
|
||||
AiterFlashAttentionMetadataBuilder.asm_kv_scale_tables, AsmKvScaleTables
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user