[ROCm] Pad non-aligned AITER MLA heads (#51647)

Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
yinfengLiu
2026-08-18 16:37:10 -05:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 12f64b39d2
commit 5d8a4cf976
4 changed files with 199 additions and 57 deletions
@@ -1,13 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Head-count padding + kernel selection for the ROCm AITER MLA backend.
"""Head padding and kernel selection for the ROCm AITER MLA backend.
Kimi-K3 at TP8 puts 12 heads/rank and at TP16 puts 6 heads/rank on the AITER
MLA decode. The asm persistent decode requires exactly 16 heads, so small
head counts are tile-padded to 16 and the padding heads are sliced back off
the output. Divisor counts (1/2/4/8) may keep the Gluon kernel, but only on
gfx950 where that kernel has a build; every small head count on gfx942 (which
has no Gluon build) is routed to the asm persistent decode instead.
The asm persistent decode requires a 16-aligned head count, so unaligned
counts through 128 are tile-padded to the next multiple of 16 and sliced back
off the output. Small divisor counts (1/2/4/8) preserve their existing
repeat-interleave path and may keep the Gluon kernel on gfx950.
"""
import math
@@ -19,6 +17,7 @@ import torch.nn.functional as F
from vllm._aiter_ops import is_aiter_found
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla import rocm_aiter_mla
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLADecodeMetadata,
AiterMLAHelper,
@@ -33,12 +32,20 @@ QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
CONTEXT_LEN = 4096
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
# Non-divisor counts must go through the tile-and-slice path; divisor counts
# (of 16) keep repeat_interleave. Both must pad to exactly 16 and round-trip.
# Small non-divisor counts use tile-and-slice; divisors of 16 keep
# repeat_interleave. Both pad to exactly 16 and round-trip.
NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15]
DIVISOR_HEADS = [1, 2, 4, 8]
@pytest.fixture(autouse=True)
def _disable_native_h24(monkeypatch):
"""Exercise the padding fallback unless a test explicitly enables H24."""
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False
)
def _rocm_aiter_available() -> bool:
return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available()
@@ -123,6 +130,72 @@ def test_h12_output_discards_padding_heads():
torch.testing.assert_close(unpadded_o, o[:, :12])
def test_h24_query_is_tile_padded_to_h32():
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
assert padded_q.shape == (2, 32, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(padded_q[:, :24], q)
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
def test_h24_output_discards_h32_padding_heads():
o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o)
assert unpadded_o.shape == (2, 24, 4)
torch.testing.assert_close(unpadded_o, o[:, :24])
def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch):
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False
)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_native_h24_supported",
lambda: (
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
),
)
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
assert padded_q.shape == (2, 32, 4)
torch.testing.assert_close(padded_q[:, :24], q)
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
def test_native_h24_requires_reducer_and_metadata(monkeypatch):
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla,
"_aiter_mla_native_h24_supported",
lambda: (
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
),
)
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24
assert AiterMLAHelper.get_mla_padded_q(24, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q
def test_existing_divisor_head_mapping_is_unchanged():
q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4)
@@ -134,6 +207,18 @@ def test_existing_divisor_head_mapping_is_unchanged():
torch.testing.assert_close(unpadded_o, q)
@pytest.mark.parametrize("num_heads", [17, 24, 31])
def test_unaligned_head_counts_round_trip_through_h32(num_heads: int):
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
assert padded_q.shape == (2, 32, 4)
assert padded_q.is_contiguous()
torch.testing.assert_close(unpadded_o, q)
@pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS)
def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
@@ -147,26 +232,30 @@ def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
torch.testing.assert_close(unpadded_o, q)
def test_num_heads_ge_16_is_passthrough():
q = torch.arange(2 * 16 * 4, dtype=torch.float32).view(2, 16, 4)
assert AiterMLAHelper.get_mla_padded_q(16, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(16, q) is q
def test_aligned_h32_is_zero_copy():
q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
assert AiterMLAHelper.get_mla_padded_q(32, q) is q
assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q
def test_is_valid_num_heads():
for n in range(1, 16):
for n in range(1, 129):
assert AiterMLAHelper.is_valid_num_heads(n)
assert AiterMLAHelper.is_valid_num_heads(16)
assert AiterMLAHelper.is_valid_num_heads(32)
assert AiterMLAHelper.is_valid_num_heads(24)
assert AiterMLAHelper.is_valid_num_heads(127)
# Aligned counts remain valid above the range where padding is supported.
assert AiterMLAHelper.is_valid_num_heads(144)
assert not AiterMLAHelper.is_valid_num_heads(0)
assert not AiterMLAHelper.is_valid_num_heads(129)
def test_nondivisor_and_multitoken_never_use_gluon():
# Non-divisor decode always takes the asm path (12 heads/rank at TP8).
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto")
# >=16 heads never pad, never Gluon.
# >=16 heads never use Gluon, including unaligned counts padded for asm.
assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto")
assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto")
# Multi-token (verify / qlen>1) is never the single-token Gluon decode.
assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto")
assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto")
@@ -15,6 +15,7 @@ if not current_platform.is_rocm():
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
AiterMLAHelper,
AiterMLAMetadataBuilder,
)
@@ -83,7 +84,7 @@ def _builder(
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False,
kernel_block_size=kernel_block_size,
_num_attention_heads=16,
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
@@ -109,7 +110,7 @@ def test_backend_declares_uniform_batch_support():
)
@pytest.mark.parametrize("num_heads", [8, 16, 32, 64, 128])
@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128])
@pytest.mark.parametrize(
"spec_method, parallel_drafting",
[
@@ -127,8 +128,8 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
):
"""Aiter init sizes the metadata for every query length decode can be handed.
Sweeping num_heads asserts the max(16, num_heads) clamp is what sizes the
metadata, covering the fp8 nhead=32 (TP4) fold path.
Sweeping num_heads asserts metadata is sized for the padded decode shape,
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
"""
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
@@ -207,7 +208,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
{
"max_batch_size": config.scheduler_config.max_num_seqs,
"max_qo_len": builder.reorder_batch_threshold,
"num_attention_heads": max(16, num_heads),
"num_attention_heads": AiterMLAHelper.get_actual_mla_num_heads(num_heads),
"q_dtype": dtypes.fp8,
"kv_dtype": dtypes.fp8,
"is_sparse": False,
@@ -381,6 +382,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch):
(1, 1, 16, "auto", True), # non-MTP decode
(4, 2, 16, "auto", True), # MTP deployment, in-range step
(4, 4, 16, "auto", True), # MTP deployment, full-qlen verification step
(1, 1, 24, "auto", True), # unaligned H24 pads to H32 persistent decode
(2, 4, 16, "auto", False), # step demand exceeds provisioned K -> fallback
(1, 1, 8, "auto", False), # divisor head count -> Gluon decode owns qlen==1
(4, 4, 8, "auto", False), # small head count -> Gluon flatten owns qlen>1
@@ -410,7 +412,8 @@ def test_persistent_metadata_gate(
K = _mtp_decode_qlen sizes the metadata buffers at init; a decode step gets
the pre-built schedule only when its qlen fits those buffers, otherwise it
falls back to the kernel computing its own. qlen==1 (non-MTP) must stay
in-range -- dropping it is the regression this guards.
in-range -- dropping it is the regression this guards. This includes
unaligned H24, whose padded H32 decode uses persistent metadata.
Only the Gluon paths ignore the schedule, so the gate follows the routing
predicates rather than the raw head count. Reading `num_heads >= 16` instead
@@ -3,6 +3,7 @@
import functools
from dataclasses import dataclass
from pathlib import Path
from typing import ClassVar, Final
import torch
@@ -77,6 +78,48 @@ def _fp8_mla_prefill_supported() -> bool:
return True
@functools.lru_cache(maxsize=1)
def _aiter_mla_native_h24_reducer_supported() -> bool:
"""Whether AITER's JIT reducer supports the native H24/512 shape."""
try:
from aiter.jit.core import AITER_CSRC_DIR
reduce_source = Path(AITER_CSRC_DIR) / "kernels" / "mla" / "reduce.cu"
source = "".join(reduce_source.read_text(encoding="utf-8").split())
except (ImportError, OSError):
return False
return "MLA_REDUCE_CASE_EF(NUM_HEAD,24,HEAD_DIM,512," in source
@functools.lru_cache(maxsize=1)
def _aiter_mla_native_h24_metadata_supported() -> bool:
"""Whether AITER's fast MLA metadata planner accepts native H24.
The reducer and metadata planner have independent shape dispatch. Checking
only the reducer can route H24 into a planner that rejects it before the
attention kernel launches. Until AITER exposes a capability API, inspect
the shipped JIT source for an explicit native-H24 planner branch.
"""
try:
from aiter.jit.core import AITER_CSRC_DIR
metadata_source = (
Path(AITER_CSRC_DIR) / "kernels" / "mla" / "metadata" / "v1_2_device.cuh"
)
source = "".join(metadata_source.read_text(encoding="utf-8").split())
except (ImportError, OSError):
return False
return "num_heads==24" in source
def _aiter_mla_native_h24_supported() -> bool:
"""Whether the complete AITER decode path supports native H24."""
return (
_aiter_mla_native_h24_reducer_supported()
and _aiter_mla_native_h24_metadata_supported()
)
@functools.lru_cache(maxsize=1)
def _gluon_mla_decode_supported() -> bool:
"""The small-head Gluon MLA decode kernel only has a gfx950 (CDNA4) build.
@@ -308,10 +351,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
from aiter import dtypes, get_mla_metadata_info_v1
# For num_attention_heads < 16 (e.g. kimi-k2.5 head=8 with TP8),
# make sure get_mla_metadata_info_v1 / get_mla_metadata_v1 are consistent
# with the actual tensor shape passed to mla_decode_fwd.
self._num_attention_heads = max(16, self.num_heads)
# Keep metadata sizing consistent with the padded tensor shape passed
# to mla_decode_fwd.
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
self.num_heads
)
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
kv_cache_dtype_str = "fp8"
@@ -839,14 +883,16 @@ def _expand_page_indices_kernel(
class AiterMLAHelper:
"""
AITER MLA persistent (asm) decode requires num_heads >= 16. Head counts
< 16 are padded up to exactly 16: divisors of 16 by repeat_interleave,
other counts (e.g. 12 heads/rank at TP8, 6 at TP16) by tiling the query
heads and slicing to 16. Non-divisor padded decodes take the asm path;
divisors and max_qo_len > 1 small-head verify still use Gluon.
AITER MLA persistent (asm) decode requires a multiple of 16 heads. Unaligned
head counts through 128 are padded to the next multiple of 16 by tiling the
query heads and slicing to the padded size. Native H24 AITER builds bypass
that padding. Small divisors of 16 retain the existing repeat_interleave and
strided-unpad behavior. Native and aligned counts pass through without
copies.
"""
_AITER_MIN_MLA_HEADS: Final = 16
_AITER_MAX_PADDED_MLA_HEADS: Final = 128
# Largest qlen the padded gqa=16 asm decode has a bf16 persistent kernel
# for. Above it only the non-persistent qseqlen=8 entry exists, and the
# fold that reaches a persistent one is gfx950-only.
@@ -856,48 +902,50 @@ class AiterMLAHelper:
@staticmethod
def check_num_heads_validity(num_heads: int):
assert AiterMLAHelper.is_valid_num_heads(num_heads), (
"ROCM AITER MLA requires 1-15 heads (padded to 16 for asm "
"persistent decode; exact divisors of 16 may keep Gluon) or a "
f"multiple of 16 heads, but got {num_heads}.\n"
"ROCM AITER MLA requires a positive multiple of 16 heads, or an "
"unaligned head count up to 128 (padded to the next multiple of "
f"16), but got {num_heads}.\n"
f"Try adjusting tensor_parallel_size value."
)
@staticmethod
def is_valid_num_heads(num_heads: int) -> bool:
return num_heads > 0 and (
num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS
or num_heads % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
return (
num_heads > 0
and num_heads not in AiterMLAHelper._AITER_UNSUPPORTED_HEADS
and (
num_heads <= AiterMLAHelper._AITER_MAX_PADDED_MLA_HEADS
or num_heads % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
)
)
@staticmethod
def get_actual_mla_num_heads(num_heads: int) -> int:
return max(num_heads, AiterMLAHelper._AITER_MIN_MLA_HEADS)
if num_heads == 24 and _aiter_mla_native_h24_supported():
return num_heads
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
return -(-num_heads // m) * m
@staticmethod
def get_mla_padded_q(num_heads: int, q: torch.Tensor) -> torch.Tensor:
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
if num_heads >= m:
m = AiterMLAHelper.get_actual_mla_num_heads(num_heads)
if num_heads == m:
return q
if m % num_heads == 0:
return q.repeat_interleave(m // num_heads, dim=1)
# Non-divisor head counts (e.g. 12 heads/rank at TP8, 6 at TP16) cannot
# be padded by repeat_interleave. Tile the query heads and slice to
# exactly m; this reaches m for any 0 < num_heads < m (unlike a single
# append, which under-pads when num_heads < m - num_heads). MLA
# attention is independent per query head over the shared KV, so the
# padding heads cannot affect heads [0:num_heads]; they are sliced back
# off in get_mla_unpadded_o.
# Non-divisor head counts cannot be padded by repeat_interleave. Tile
# the query heads and slice to exactly m. MLA attention is independent
# per query head over the shared KV, so padding heads cannot affect
# heads [0:num_heads]; they are sliced back off the output.
reps = -(-m // num_heads) # ceil(m / num_heads)
# Slicing a tiled tensor down to m yields a non-contiguous view whenever
# reps * num_heads > m (the common case: TP8 12->24->16, TP16 6->18->16).
# The asm persistent decode reads q as a packed [tokens, m, head_dim]
# buffer, so materialize a contiguous copy. No-op when already contiguous.
# Slicing a tiled tensor yields a non-contiguous view. The asm decode
# reads q as packed [tokens, m, head_dim], so materialize it.
return q.repeat(1, reps, 1)[:, :m, :].contiguous()
@staticmethod
def get_mla_unpadded_o(num_heads: int, o: torch.Tensor) -> torch.Tensor:
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
if num_heads >= m:
m = AiterMLAHelper.get_actual_mla_num_heads(num_heads)
if num_heads == m:
return o
if m % num_heads == 0:
return o[:, :: m // num_heads, :]
@@ -422,9 +422,11 @@ class ROCMAiterMLASparseMetadataBuilder(
# so the buffers are large enough for any decode shape we might see.
from aiter import dtypes, get_mla_metadata_info_v1
# Aiter sparse MLA also requires num_heads >= 16 (will be padded by
# AiterMLAHelper.get_mla_padded_q in forward).
self._num_attention_heads = max(16, self.num_heads)
# Keep metadata sizing consistent with the padded tensor shape passed
# to the sparse decode kernel.
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
self.num_heads
)
q_dtype = self.model_dtype
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")