[Model][Hardware][AMD][Kernel] Part 2/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_ATTN (#42755)

Signed-off-by: Jack Hu <Jack.Hu@amd.com>
Signed-off-by: Jack Hu <jingxujaster@hotmail.com>
Signed-off-by: Jack Hu <jack.hu@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Jack Hu
2026-10-15 17:10:14 -07:00
committed by GitHub
co-authored by Douglas Lehr
parent af28c171f5
commit ef38e8d006
23 changed files with 701 additions and 205 deletions
@@ -157,10 +157,17 @@ class QKNormRoPEKVCacheTestModel(torch.nn.Module):
)
if self.kv_cache_dtype != self.dtype:
self.attn._k_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
self.attn._v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
self.attn._k_scale_float = 1.0
self.attn._v_scale_float = 1.0
# Non-unit scales so the fused and unfused writers must agree on them.
self.attn._k_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
self.attn._v_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
self.attn._k_scale_cpu = torch.tensor(
0.5, dtype=torch.float32, device="cpu"
)
self.attn._v_scale_cpu = torch.tensor(
0.25, dtype=torch.float32, device="cpu"
)
self.attn._k_scale_float = 0.5
self.attn._v_scale_float = 0.25
else:
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
@@ -527,16 +534,6 @@ def _run_qk_norm_rope_kvcache_fusion_test(
else:
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
if not expect_fusion or (not is_fp8_cache and mrope_section is None):
# The AITER PTS kernel populates k_out only for non-FP8 caches.
# With FP8, the kernel writes quantized K directly to the cache
# and may leave k_out uninitialised. In production this is fine
# because downstream attention reads K from the cache.
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
# Should be bit exact since no processing had been done on v for both paths
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
# Allow one E4M3 quantization step while still comparing every block.
if is_fp8_cache:
@@ -544,6 +541,18 @@ def _run_qk_norm_rope_kvcache_fusion_test(
else:
cache_atol, cache_rtol = ATOL, RTOL
# The fused k_out is the dequantized cache write, so it carries the same
# rounding; FA and ROCM_ATTN prefill consume it directly. Unified reads K
# from the cache and asks the kernel not to write k_out at all.
if attn_backend != AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:
torch.testing.assert_close(
k_unfused, k_fused, atol=cache_atol, rtol=cache_rtol
)
# Should be bit exact since no processing had been done on v for both paths
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
# Whole cache, so the interleaved V write (ROCM_ATTN kv_cache[1]) is checked.
torch.testing.assert_close(
kv_cache_unfused.float(),
kv_cache_fused.float(),
@@ -570,20 +579,52 @@ _FUSION_CONFIGS = [
_FUSION_CONFIGS,
)
@pytest.mark.parametrize(
"attn_backend",
"attn_backend, use_shuffle_kv_layout, kv_layout",
[
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
AttentionBackendEnum.ROCM_AITER_FA,
# ROCM_ATTN always writes the shuffled, interleaved-V layout and needs each
# K/V side contiguous per block, so head-major only.
pytest.param(
AttentionBackendEnum.ROCM_ATTN,
"0",
KVCacheLayout.LBHNC,
id="rocm_attn",
),
# Unified never reads the shuffle env; its strided views take any layout.
pytest.param(
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"0",
KVCacheLayout.LBHNC,
id="unified-head_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"0",
KVCacheLayout.LBNHC,
id="unified-token_major",
),
# FA takes any layout unshuffled; its shuffle writer needs each K/V side
# contiguous per block, so head-major only.
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"0",
KVCacheLayout.LBHNC,
id="fa-head_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"0",
KVCacheLayout.LBNHC,
id="fa-token_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"1",
KVCacheLayout.LBHNC,
id="fa-shuffle",
),
],
)
@pytest.mark.parametrize("num_tokens", [5, 2048])
@pytest.mark.parametrize(
"kv_layout",
[
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
],
)
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("block_size", [16, 32, 64])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@@ -615,6 +656,7 @@ def test_qk_norm_rope_kvcache_fusion(
kv_cache_dtype: CacheDType,
rms_norm_eps: float,
custom_op: str,
use_shuffle_kv_layout: str,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
@@ -627,7 +669,7 @@ def test_qk_norm_rope_kvcache_fusion(
rotary_dim=rotary_dim,
block_size=block_size,
is_neox=is_neox,
use_shuffle_kv_layout="0",
use_shuffle_kv_layout=use_shuffle_kv_layout,
kv_layout=kv_layout,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
+25 -1
View File
@@ -122,6 +122,7 @@ def ref_single_query_cached_kv_attention(
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("use_interleaved_v_cache", [False, True])
def test_paged_attention(
kv_cache_factory,
num_seqs: int,
@@ -133,9 +134,14 @@ def test_paged_attention(
kv_cache_dtype: str,
seed: int,
device: str,
use_interleaved_v_cache: bool,
) -> None:
if current_platform.is_navi() and (
kv_cache_dtype == "fp8" or head_size != 128 or block_size != 16 or use_alibi
kv_cache_dtype == "fp8"
or head_size != 128
or block_size != 16
or use_alibi
or use_interleaved_v_cache
):
pytest.skip()
@@ -201,6 +207,20 @@ def test_paged_attention(
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
standard_value_cache = value_cache
if use_interleaved_v_cache:
# Hand the kernel V in the interleaved layout (same 4D shape); the
# reference below gets the standard layout back.
x_v = 16 // value_cache.element_size()
value_cache = (
value_cache.view(
NUM_BLOCKS, num_kv_heads, head_size, block_size // x_v, x_v
)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(NUM_BLOCKS, num_kv_heads, head_size, block_size)
)
# Call the paged attention kernel.
output = torch.empty_like(query)
num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
@@ -234,6 +254,7 @@ def test_paged_attention(
kv_cache_dtype,
k_scale,
v_scale,
use_interleaved_v_cache=use_interleaved_v_cache,
)
opcheck(
@@ -259,10 +280,13 @@ def test_paged_attention(
v_scale,
None,
"f16",
use_interleaved_v_cache,
),
cond=(head_size == 64 and block_size == BLOCK_SIZES[0]),
)
value_cache = standard_value_cache
# Run the reference implementation.
if kv_cache_dtype == "fp8":
# Convert cache data back to dtype.
+90 -2
View File
@@ -109,6 +109,7 @@ def create_alibi_causal_mask(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention(
num_heads: int,
@@ -119,6 +120,7 @@ def test_contexted_kv_attention(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
block_size: int = 32,
) -> None:
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
@@ -225,6 +227,23 @@ def test_contexted_kv_attention(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
extra_op_kwargs = {}
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
# read it back interleaved.
x_v = 16 // v_cache.element_size()
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
if op is context_attention_fwd:
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
else:
extra_op_kwargs["use_interleaved_v_cache"] = True
# Warm up the Triton kernel by calling it once before actually measuring
# generation time
op(
@@ -243,6 +262,7 @@ def test_contexted_kv_attention(
k_scale,
v_scale,
sliding_window=sliding_window,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
start_time = time.time()
@@ -262,6 +282,7 @@ def test_contexted_kv_attention(
k_scale,
v_scale,
sliding_window=sliding_window,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
end_time = time.time()
@@ -386,6 +407,7 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_cached_kv(
num_heads: int,
@@ -394,6 +416,7 @@ def test_contexted_kv_attention_cached_kv(
dtype: torch.dtype,
kv_cache_dtype: str,
device: str,
interleave_v: bool,
block_size: int = 32,
) -> None:
# Exercises the KV_FROM_CACHE path of context_attention_fwd: the current
@@ -487,6 +510,18 @@ def test_contexted_kv_attention_cached_kv(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
interleaved_v_pack_factor = 0
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape).
interleaved_v_pack_factor = 16 // v_cache.element_size()
x_v = interleaved_v_pack_factor
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
# Cached-K/V path: current-chunk k and v are None.
context_attention_fwd(
query,
@@ -504,6 +539,7 @@ def test_contexted_kv_attention_cached_kv(
k_scale,
v_scale,
sliding_window=0,
interleaved_v_pack_factor=interleaved_v_pack_factor,
)
torch.accelerator.synchronize()
@@ -544,8 +580,11 @@ def test_contexted_kv_attention_cached_kv(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> None:
def test_contexted_kv_attention_cached_kv_block_table_boundary(
device: str, interleave_v: bool
) -> None:
# Boundary guard for the KV_FROM_CACHE block-table load. With an
# exact-sized block table (row length == number of blocks for the
# sequence) and a query that ends the sequence, the last K/V tile has
@@ -611,6 +650,18 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
interleaved_v_pack_factor = 0
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape).
interleaved_v_pack_factor = 16 // v_cache.element_size()
x_v = interleaved_v_pack_factor
v_cache = (
v_cache.view(num_blocks, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(num_blocks, num_kv_heads, head_size, block_size)
)
# Cached-K/V path: current-chunk k and v are None.
context_attention_fwd(
query,
@@ -628,6 +679,7 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
k_scale,
v_scale,
sliding_window=0,
interleaved_v_pack_factor=interleaved_v_pack_factor,
)
torch.accelerator.synchronize()
@@ -723,6 +775,7 @@ def test_contexted_kv_attention_cached_kv_alibi_unsupported(device: str) -> None
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_alibi(
num_heads: int,
@@ -732,6 +785,7 @@ def test_contexted_kv_attention_alibi(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
block_size: int = 32,
) -> None:
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
@@ -860,6 +914,23 @@ def test_contexted_kv_attention_alibi(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
extra_op_kwargs = {}
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
# read it back interleaved.
x_v = 16 // v_cache.element_size()
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
if op is context_attention_fwd:
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
else:
extra_op_kwargs["use_interleaved_v_cache"] = True
# Warm up the Triton kernel by calling it once before actually measuring
# generation time
op(
@@ -878,6 +949,7 @@ def test_contexted_kv_attention_alibi(
k_scale,
v_scale,
alibi_slopes=alibi_slopes,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
start_time = time.time()
@@ -897,6 +969,7 @@ def test_contexted_kv_attention_alibi(
k_scale,
v_scale,
alibi_slopes=alibi_slopes,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
end_time = time.time()
@@ -992,6 +1065,7 @@ def test_contexted_kv_attention_alibi(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_f32(
num_heads: int,
@@ -1002,6 +1076,7 @@ def test_contexted_kv_attention_f32(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
test_contexted_kv_attention(
num_heads,
@@ -1012,6 +1087,7 @@ def test_contexted_kv_attention_f32(
kv_cache_dtype,
device,
op,
interleave_v=interleave_v,
)
@@ -1023,6 +1099,7 @@ def test_contexted_kv_attention_f32(
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_alibi_f32(
num_heads: int,
@@ -1032,9 +1109,17 @@ def test_contexted_kv_attention_alibi_f32(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
test_contexted_kv_attention_alibi(
num_heads, num_queries_per_kv, head_size, dtype, kv_cache_dtype, device, op
num_heads,
num_queries_per_kv,
head_size,
dtype,
kv_cache_dtype,
device,
op,
interleave_v=interleave_v,
)
@@ -1052,6 +1137,7 @@ NONSTANDARD_BLOCK_SIZE_SHAPES = [
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_qwen3_nonstandard_block_size(
num_heads: int,
@@ -1061,6 +1147,7 @@ def test_qwen3_nonstandard_block_size(
dtype: torch.dtype,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
"""Non-power-of-2 pages must match, even when a tile straddles a page."""
if not current_platform.is_rocm():
@@ -1076,4 +1163,5 @@ def test_qwen3_nonstandard_block_size(
kv_cache_dtype="auto",
device=device,
op=op,
interleave_v=interleave_v,
)
@@ -1061,3 +1061,25 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
rtol=rtol,
)
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
def test_asm_kv_scale_tables_shared_across_builders():
"""Builders share one AsmKvScaleTables; equal (shape, scale) requests must
return the same dense table, a different scale a different one."""
from vllm.v1.attention.backends.rocm_aiter_fa import (
AiterFlashAttentionMetadataBuilder,
AsmKvScaleTables,
)
tables = AsmKvScaleTables()
device = torch.device("cuda")
table = tables.get(device, 8, 2, 16, 0.5)
assert tables.get(device, 8, 2, 16, 0.5) is table
assert tables.get(device, 8, 2, 16, 1.0) is not table
assert table.shape == (8, 2, 16)
assert table.dtype == torch.float32
assert torch.all(table == 0.5)
# The builder class holds the single process-wide instance.
assert isinstance(
AiterFlashAttentionMetadataBuilder.asm_kv_scale_tables, AsmKvScaleTables
)
+3 -1
View File
@@ -368,9 +368,11 @@ async def test_dp_pause_late_request_does_not_block_drain():
# down first. The front-end sets its own copy optimistically when it
# forwards the wake, so sample only after several coordinator
# publishes (every 100ms while stats change) have overwritten it.
# Rank 0 reports the wave only after its first step, which can outlast
# the 2s on a cold device, so wait for the report rather than sample it.
await asyncio.sleep(2)
assert not long_request.done(), "the warmup request was too short to sample"
assert engine.engine_core.dp_engines_running(), (
assert await _poll_flag(engine, True, timeout=30), (
"the coordinator does not report the engines as running while they are"
)