[ROCm] Pad non-aligned AITER MLA heads (#51647)

Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
yinfengLiu
2026-08-18 16:37:10 -05:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 12f64b39d2
commit 5d8a4cf976
4 changed files with 199 additions and 57 deletions
@@ -1,13 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Head-count padding + kernel selection for the ROCm AITER MLA backend.
"""Head padding and kernel selection for the ROCm AITER MLA backend.
Kimi-K3 at TP8 puts 12 heads/rank and at TP16 puts 6 heads/rank on the AITER
MLA decode. The asm persistent decode requires exactly 16 heads, so small
head counts are tile-padded to 16 and the padding heads are sliced back off
the output. Divisor counts (1/2/4/8) may keep the Gluon kernel, but only on
gfx950 where that kernel has a build; every small head count on gfx942 (which
has no Gluon build) is routed to the asm persistent decode instead.
The asm persistent decode requires a 16-aligned head count, so unaligned
counts through 128 are tile-padded to the next multiple of 16 and sliced back
off the output. Small divisor counts (1/2/4/8) preserve their existing
repeat-interleave path and may keep the Gluon kernel on gfx950.
"""
import math
@@ -19,6 +17,7 @@ import torch.nn.functional as F
from vllm._aiter_ops import is_aiter_found
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLADecodeMetadata,
AiterMLAHelper,
@@ -33,12 +32,20 @@ QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
CONTEXT_LEN = 4096
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
# Non-divisor counts must go through the tile-and-slice path; divisor counts
# (of 16) keep repeat_interleave. Both must pad to exactly 16 and round-trip.
# Small non-divisor counts use tile-and-slice; divisors of 16 keep
# repeat_interleave. Both pad to exactly 16 and round-trip.
NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15]
DIVISOR_HEADS = [1, 2, 4, 8]
@pytest.fixture(autouse=True)
def _disable_native_h24(monkeypatch):
"""Exercise the padding fallback unless a test explicitly enables H24."""
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False
)
def _rocm_aiter_available() -> bool:
return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available()
@@ -123,6 +130,72 @@ def test_h12_output_discards_padding_heads():
torch.testing.assert_close(unpadded_o, o[:, :12])
def test_h24_query_is_tile_padded_to_h32():
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
assert padded_q.shape == (2, 32, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(padded_q[:, :24], q)
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
def test_h24_output_discards_h32_padding_heads():
o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o)
assert unpadded_o.shape == (2, 24, 4)
torch.testing.assert_close(unpadded_o, o[:, :24])
def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch):
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False
)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_native_h24_supported",
lambda: (
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
),
)
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
assert padded_q.shape == (2, 32, 4)
torch.testing.assert_close(padded_q[:, :24], q)
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
def test_native_h24_requires_reducer_and_metadata(monkeypatch):
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_native_h24_supported",
lambda: (
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
),
)
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24
assert AiterMLAHelper.get_mla_padded_q(24, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q
def test_existing_divisor_head_mapping_is_unchanged():
q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4)
@@ -134,6 +207,18 @@ def test_existing_divisor_head_mapping_is_unchanged():
torch.testing.assert_close(unpadded_o, q)
@pytest.mark.parametrize("num_heads", [17, 24, 31])
def test_unaligned_head_counts_round_trip_through_h32(num_heads: int):
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
assert padded_q.shape == (2, 32, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(unpadded_o, q)
@pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS)
def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
@@ -147,26 +232,30 @@ def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
torch.testing.assert_close(unpadded_o, q)
def test_num_heads_ge_16_is_passthrough():
q = torch.arange(2 * 16 * 4, dtype=torch.float32).view(2, 16, 4)
assert AiterMLAHelper.get_mla_padded_q(16, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(16, q) is q
def test_aligned_h32_is_zero_copy():
q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
assert AiterMLAHelper.get_mla_padded_q(32, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q
def test_is_valid_num_heads():
for n in range(1, 16):
for n in range(1, 129):
assert AiterMLAHelper.is_valid_num_heads(n)
assert AiterMLAHelper.is_valid_num_heads(16)
assert AiterMLAHelper.is_valid_num_heads(32)
assert AiterMLAHelper.is_valid_num_heads(24)
assert AiterMLAHelper.is_valid_num_heads(127)
# Aligned counts remain valid above the range where padding is supported.
assert AiterMLAHelper.is_valid_num_heads(144)
assert not AiterMLAHelper.is_valid_num_heads(0)
assert not AiterMLAHelper.is_valid_num_heads(129)
def test_nondivisor_and_multitoken_never_use_gluon():
# Non-divisor decode always takes the asm path (12 heads/rank at TP8).
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto")
# >=16 heads never pad, never Gluon.
# >=16 heads never use Gluon, including unaligned counts padded for asm.
assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto")
assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto")
# Multi-token (verify / qlen>1) is never the single-token Gluon decode.
assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto")
assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto")
@@ -15,6 +15,7 @@ if not current_platform.is_rocm():
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
AiterMLAHelper,
AiterMLAMetadataBuilder,
)
@@ -83,7 +84,7 @@ def _builder(
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False,
kernel_block_size=kernel_block_size,
_num_attention_heads=16,
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
@@ -109,7 +110,7 @@ def test_backend_declares_uniform_batch_support():
)
@pytest.mark.parametrize("num_heads", [8, 16, 32, 64, 128])
@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128])
@pytest.mark.parametrize(
"spec_method, parallel_drafting",
[
@@ -127,8 +128,8 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
):
"""Aiter init sizes the metadata for every query length decode can be handed.
Sweeping num_heads asserts the max(16, num_heads) clamp is what sizes the
metadata, covering the fp8 nhead=32 (TP4) fold path.
Sweeping num_heads asserts metadata is sized for the padded decode shape,
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
"""
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
@@ -207,7 +208,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
{
"max_batch_size": config.scheduler_config.max_num_seqs,
"max_qo_len": builder.reorder_batch_threshold,
"num_attention_heads": max(16, num_heads),
"num_attention_heads": AiterMLAHelper.get_actual_mla_num_heads(num_heads),
"q_dtype": dtypes.fp8,
"kv_dtype": dtypes.fp8,
"is_sparse": False,
@@ -381,6 +382,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch):
(1, 1, 16, "auto", True), # non-MTP decode
(4, 2, 16, "auto", True), # MTP deployment, in-range step
(4, 4, 16, "auto", True), # MTP deployment, full-qlen verification step
(1, 1, 24, "auto", True), # unaligned H24 pads to H32 persistent decode
(2, 4, 16, "auto", False), # step demand exceeds provisioned K -> fallback
(1, 1, 8, "auto", False), # divisor head count -> Gluon decode owns qlen==1
(4, 4, 8, "auto", False), # small head count -> Gluon flatten owns qlen>1
@@ -410,7 +412,8 @@ def test_persistent_metadata_gate(
K = _mtp_decode_qlen sizes the metadata buffers at init; a decode step gets
the pre-built schedule only when its qlen fits those buffers, otherwise it
falls back to the kernel computing its own. qlen==1 (non-MTP) must stay
in-range -- dropping it is the regression this guards.
in-range -- dropping it is the regression this guards. This includes
unaligned H24, whose padded H32 decode uses persistent metadata.
Only the Gluon paths ignore the schedule, so the gate follows the routing
predicates rather than the raw head count. Reading `num_heads >= 16` instead