diff --git a/tests/kernels/attention/test_rocm_aiter_mla_head_padding.py b/tests/kernels/attention/test_rocm_aiter_mla_head_padding.py index 744f1852ba..8a284e320a 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_head_padding.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_head_padding.py @@ -1,13 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Head-count padding + kernel selection for the ROCm AITER MLA backend. +"""Head padding and kernel selection for the ROCm AITER MLA backend. -Kimi-K3 at TP8 puts 12 heads/rank and at TP16 puts 6 heads/rank on the AITER -MLA decode. The asm persistent decode requires exactly 16 heads, so small -head counts are tile-padded to 16 and the padding heads are sliced back off -the output. Divisor counts (1/2/4/8) may keep the Gluon kernel, but only on -gfx950 where that kernel has a build; every small head count on gfx942 (which -has no Gluon build) is routed to the asm persistent decode instead. +The asm persistent decode requires a 16-aligned head count, so unaligned +counts through 128 are tile-padded to the next multiple of 16 and sliced back +off the output. Small divisor counts (1/2/4/8) preserve their existing +repeat-interleave path and may keep the Gluon kernel on gfx950. """ import math @@ -19,6 +17,7 @@ import torch.nn.functional as F from vllm._aiter_ops import is_aiter_found from vllm.platforms import current_platform +from vllm.v1.attention.backends.mla import rocm_aiter_mla from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( AiterMLADecodeMetadata, AiterMLAHelper, @@ -33,12 +32,20 @@ QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM CONTEXT_LEN = 4096 SCALE = 1.0 / math.sqrt(QK_HEAD_DIM) -# Non-divisor counts must go through the tile-and-slice path; divisor counts -# (of 16) keep repeat_interleave. Both must pad to exactly 16 and round-trip. +# Small non-divisor counts use tile-and-slice; divisors of 16 keep +# repeat_interleave. Both pad to exactly 16 and round-trip. NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15] DIVISOR_HEADS = [1, 2, 4, 8] +@pytest.fixture(autouse=True) +def _disable_native_h24(monkeypatch): + """Exercise the padding fallback unless a test explicitly enables H24.""" + monkeypatch.setattr( + rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False + ) + + def _rocm_aiter_available() -> bool: return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available() @@ -123,6 +130,72 @@ def test_h12_output_discards_padding_heads(): torch.testing.assert_close(unpadded_o, o[:, :12]) +def test_h24_query_is_tile_padded_to_h32(): + q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4) + + padded_q = AiterMLAHelper.get_mla_padded_q(24, q) + + assert padded_q.shape == (2, 32, 4) + assert padded_q.is_contiguous() + torch.testing.assert_close(padded_q[:, :24], q) + torch.testing.assert_close(padded_q[:, 24:], q[:, :8]) + + +def test_h24_output_discards_h32_padding_heads(): + o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4) + + unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o) + + assert unpadded_o.shape == (2, 24, 4) + torch.testing.assert_close(unpadded_o, o[:, :24]) + + +def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch): + monkeypatch.setattr( + rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True + ) + monkeypatch.setattr( + rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False + ) + monkeypatch.setattr( + rocm_aiter_mla, + "_aiter_mla_native_h24_supported", + lambda: ( + rocm_aiter_mla._aiter_mla_native_h24_reducer_supported() + and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported() + ), + ) + q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4) + + assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32 + padded_q = AiterMLAHelper.get_mla_padded_q(24, q) + assert padded_q.shape == (2, 32, 4) + torch.testing.assert_close(padded_q[:, :24], q) + torch.testing.assert_close(padded_q[:, 24:], q[:, :8]) + + +def test_native_h24_requires_reducer_and_metadata(monkeypatch): + monkeypatch.setattr( + rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True + ) + monkeypatch.setattr( + rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True + ) + monkeypatch.setattr( + rocm_aiter_mla, + "_aiter_mla_native_h24_supported", + lambda: ( + rocm_aiter_mla._aiter_mla_native_h24_reducer_supported() + and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported() + ), + ) + q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4) + + assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24 + assert AiterMLAHelper.get_mla_padded_q(24, q) is q + assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q + + def test_existing_divisor_head_mapping_is_unchanged(): q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4) @@ -134,6 +207,18 @@ def test_existing_divisor_head_mapping_is_unchanged(): torch.testing.assert_close(unpadded_o, q) +@pytest.mark.parametrize("num_heads", [17, 24, 31]) +def test_unaligned_head_counts_round_trip_through_h32(num_heads: int): + q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4) + + padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q) + unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q) + + assert padded_q.shape == (2, 32, 4) + assert padded_q.is_contiguous() + torch.testing.assert_close(unpadded_o, q) + + @pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS) def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int): q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4) @@ -147,26 +232,30 @@ def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int): torch.testing.assert_close(unpadded_o, q) -def test_num_heads_ge_16_is_passthrough(): - q = torch.arange(2 * 16 * 4, dtype=torch.float32).view(2, 16, 4) - assert AiterMLAHelper.get_mla_padded_q(16, q) is q - assert AiterMLAHelper.get_mla_unpadded_o(16, q) is q +def test_aligned_h32_is_zero_copy(): + q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4) + assert AiterMLAHelper.get_mla_padded_q(32, q) is q + assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q def test_is_valid_num_heads(): - for n in range(1, 16): + for n in range(1, 129): assert AiterMLAHelper.is_valid_num_heads(n) - assert AiterMLAHelper.is_valid_num_heads(16) - assert AiterMLAHelper.is_valid_num_heads(32) + assert AiterMLAHelper.is_valid_num_heads(24) + assert AiterMLAHelper.is_valid_num_heads(127) + # Aligned counts remain valid above the range where padding is supported. + assert AiterMLAHelper.is_valid_num_heads(144) assert not AiterMLAHelper.is_valid_num_heads(0) + assert not AiterMLAHelper.is_valid_num_heads(129) def test_nondivisor_and_multitoken_never_use_gluon(): # Non-divisor decode always takes the asm path (12 heads/rank at TP8). assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto") assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto") - # >=16 heads never pad, never Gluon. + # >=16 heads never use Gluon, including unaligned counts padded for asm. assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto") + assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto") # Multi-token (verify / qlen>1) is never the single-token Gluon decode. assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto") assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto") diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index c5eb921a22..1894d8aa9c 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -15,6 +15,7 @@ if not current_platform.is_rocm(): from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402 from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402 + AiterMLAHelper, AiterMLAMetadataBuilder, ) @@ -83,7 +84,7 @@ def _builder( _uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len), _use_persistent_metadata=False, kernel_block_size=kernel_block_size, - _num_attention_heads=16, + _num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads), _mla_work_meta_data=torch.empty(1, dtype=torch.int32), _mla_work_info_set=torch.empty(1, dtype=torch.int32), _mla_work_indptr=torch.empty(1, dtype=torch.int32), @@ -109,7 +110,7 @@ def test_backend_declares_uniform_batch_support(): ) -@pytest.mark.parametrize("num_heads", [8, 16, 32, 64, 128]) +@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128]) @pytest.mark.parametrize( "spec_method, parallel_drafting", [ @@ -127,8 +128,8 @@ def test_mtp_builder_init_sizes_native_fp8_metadata( ): """Aiter init sizes the metadata for every query length decode can be handed. - Sweeping num_heads asserts the max(16, num_heads) clamp is what sizes the - metadata, covering the fp8 nhead=32 (TP4) fold path. + Sweeping num_heads asserts metadata is sized for the padded decode shape, + covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding. """ dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16") @@ -207,7 +208,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata( { "max_batch_size": config.scheduler_config.max_num_seqs, "max_qo_len": builder.reorder_batch_threshold, - "num_attention_heads": max(16, num_heads), + "num_attention_heads": AiterMLAHelper.get_actual_mla_num_heads(num_heads), "q_dtype": dtypes.fp8, "kv_dtype": dtypes.fp8, "is_sparse": False, @@ -381,6 +382,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch): (1, 1, 16, "auto", True), # non-MTP decode (4, 2, 16, "auto", True), # MTP deployment, in-range step (4, 4, 16, "auto", True), # MTP deployment, full-qlen verification step + (1, 1, 24, "auto", True), # unaligned H24 pads to H32 persistent decode (2, 4, 16, "auto", False), # step demand exceeds provisioned K -> fallback (1, 1, 8, "auto", False), # divisor head count -> Gluon decode owns qlen==1 (4, 4, 8, "auto", False), # small head count -> Gluon flatten owns qlen>1 @@ -410,7 +412,8 @@ def test_persistent_metadata_gate( K = _mtp_decode_qlen sizes the metadata buffers at init; a decode step gets the pre-built schedule only when its qlen fits those buffers, otherwise it falls back to the kernel computing its own. qlen==1 (non-MTP) must stay - in-range -- dropping it is the regression this guards. + in-range -- dropping it is the regression this guards. This includes + unaligned H24, whose padded H32 decode uses persistent metadata. Only the Gluon paths ignore the schedule, so the gate follows the routing predicates rather than the raw head count. Reading `num_heads >= 16` instead diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 2f832a1357..51f487947a 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -3,6 +3,7 @@ import functools from dataclasses import dataclass +from pathlib import Path from typing import ClassVar, Final import torch @@ -77,6 +78,48 @@ def _fp8_mla_prefill_supported() -> bool: return True +@functools.lru_cache(maxsize=1) +def _aiter_mla_native_h24_reducer_supported() -> bool: + """Whether AITER's JIT reducer supports the native H24/512 shape.""" + try: + from aiter.jit.core import AITER_CSRC_DIR + + reduce_source = Path(AITER_CSRC_DIR) / "kernels" / "mla" / "reduce.cu" + source = "".join(reduce_source.read_text(encoding="utf-8").split()) + except (ImportError, OSError): + return False + return "MLA_REDUCE_CASE_EF(NUM_HEAD,24,HEAD_DIM,512," in source + + +@functools.lru_cache(maxsize=1) +def _aiter_mla_native_h24_metadata_supported() -> bool: + """Whether AITER's fast MLA metadata planner accepts native H24. + + The reducer and metadata planner have independent shape dispatch. Checking + only the reducer can route H24 into a planner that rejects it before the + attention kernel launches. Until AITER exposes a capability API, inspect + the shipped JIT source for an explicit native-H24 planner branch. + """ + try: + from aiter.jit.core import AITER_CSRC_DIR + + metadata_source = ( + Path(AITER_CSRC_DIR) / "kernels" / "mla" / "metadata" / "v1_2_device.cuh" + ) + source = "".join(metadata_source.read_text(encoding="utf-8").split()) + except (ImportError, OSError): + return False + return "num_heads==24" in source + + +def _aiter_mla_native_h24_supported() -> bool: + """Whether the complete AITER decode path supports native H24.""" + return ( + _aiter_mla_native_h24_reducer_supported() + and _aiter_mla_native_h24_metadata_supported() + ) + + @functools.lru_cache(maxsize=1) def _gluon_mla_decode_supported() -> bool: """The small-head Gluon MLA decode kernel only has a gfx950 (CDNA4) build. @@ -308,10 +351,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): from aiter import dtypes, get_mla_metadata_info_v1 - # For num_attention_heads < 16 (e.g. kimi-k2.5 head=8 with TP8), - # make sure get_mla_metadata_info_v1 / get_mla_metadata_v1 are consistent - # with the actual tensor shape passed to mla_decode_fwd. - self._num_attention_heads = max(16, self.num_heads) + # Keep metadata sizing consistent with the padded tensor shape passed + # to mla_decode_fwd. + self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads( + self.num_heads + ) kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto") if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"): kv_cache_dtype_str = "fp8" @@ -839,14 +883,16 @@ def _expand_page_indices_kernel( class AiterMLAHelper: """ - AITER MLA persistent (asm) decode requires num_heads >= 16. Head counts - < 16 are padded up to exactly 16: divisors of 16 by repeat_interleave, - other counts (e.g. 12 heads/rank at TP8, 6 at TP16) by tiling the query - heads and slicing to 16. Non-divisor padded decodes take the asm path; - divisors and max_qo_len > 1 small-head verify still use Gluon. + AITER MLA persistent (asm) decode requires a multiple of 16 heads. Unaligned + head counts through 128 are padded to the next multiple of 16 by tiling the + query heads and slicing to the padded size. Native H24 AITER builds bypass + that padding. Small divisors of 16 retain the existing repeat_interleave and + strided-unpad behavior. Native and aligned counts pass through without + copies. """ _AITER_MIN_MLA_HEADS: Final = 16 + _AITER_MAX_PADDED_MLA_HEADS: Final = 128 # Largest qlen the padded gqa=16 asm decode has a bf16 persistent kernel # for. Above it only the non-persistent qseqlen=8 entry exists, and the # fold that reaches a persistent one is gfx950-only. @@ -856,48 +902,50 @@ class AiterMLAHelper: @staticmethod def check_num_heads_validity(num_heads: int): assert AiterMLAHelper.is_valid_num_heads(num_heads), ( - "ROCM AITER MLA requires 1-15 heads (padded to 16 for asm " - "persistent decode; exact divisors of 16 may keep Gluon) or a " - f"multiple of 16 heads, but got {num_heads}.\n" + "ROCM AITER MLA requires a positive multiple of 16 heads, or an " + "unaligned head count up to 128 (padded to the next multiple of " + f"16), but got {num_heads}.\n" f"Try adjusting tensor_parallel_size value." ) @staticmethod def is_valid_num_heads(num_heads: int) -> bool: - return num_heads > 0 and ( - num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS - or num_heads % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0 + return ( + num_heads > 0 + and num_heads not in AiterMLAHelper._AITER_UNSUPPORTED_HEADS + and ( + num_heads <= AiterMLAHelper._AITER_MAX_PADDED_MLA_HEADS + or num_heads % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0 + ) ) @staticmethod def get_actual_mla_num_heads(num_heads: int) -> int: - return max(num_heads, AiterMLAHelper._AITER_MIN_MLA_HEADS) + if num_heads == 24 and _aiter_mla_native_h24_supported(): + return num_heads + m = AiterMLAHelper._AITER_MIN_MLA_HEADS + return -(-num_heads // m) * m @staticmethod def get_mla_padded_q(num_heads: int, q: torch.Tensor) -> torch.Tensor: - m = AiterMLAHelper._AITER_MIN_MLA_HEADS - if num_heads >= m: + m = AiterMLAHelper.get_actual_mla_num_heads(num_heads) + if num_heads == m: return q if m % num_heads == 0: return q.repeat_interleave(m // num_heads, dim=1) - # Non-divisor head counts (e.g. 12 heads/rank at TP8, 6 at TP16) cannot - # be padded by repeat_interleave. Tile the query heads and slice to - # exactly m; this reaches m for any 0 < num_heads < m (unlike a single - # append, which under-pads when num_heads < m - num_heads). MLA - # attention is independent per query head over the shared KV, so the - # padding heads cannot affect heads [0:num_heads]; they are sliced back - # off in get_mla_unpadded_o. + # Non-divisor head counts cannot be padded by repeat_interleave. Tile + # the query heads and slice to exactly m. MLA attention is independent + # per query head over the shared KV, so padding heads cannot affect + # heads [0:num_heads]; they are sliced back off the output. reps = -(-m // num_heads) # ceil(m / num_heads) - # Slicing a tiled tensor down to m yields a non-contiguous view whenever - # reps * num_heads > m (the common case: TP8 12->24->16, TP16 6->18->16). - # The asm persistent decode reads q as a packed [tokens, m, head_dim] - # buffer, so materialize a contiguous copy. No-op when already contiguous. + # Slicing a tiled tensor yields a non-contiguous view. The asm decode + # reads q as packed [tokens, m, head_dim], so materialize it. return q.repeat(1, reps, 1)[:, :m, :].contiguous() @staticmethod def get_mla_unpadded_o(num_heads: int, o: torch.Tensor) -> torch.Tensor: - m = AiterMLAHelper._AITER_MIN_MLA_HEADS - if num_heads >= m: + m = AiterMLAHelper.get_actual_mla_num_heads(num_heads) + if num_heads == m: return o if m % num_heads == 0: return o[:, :: m // num_heads, :] diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 3f10d61053..6ae880251f 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -422,9 +422,11 @@ class ROCMAiterMLASparseMetadataBuilder( # so the buffers are large enough for any decode shape we might see. from aiter import dtypes, get_mla_metadata_info_v1 - # Aiter sparse MLA also requires num_heads >= 16 (will be padded by - # AiterMLAHelper.get_mla_padded_q in forward). - self._num_attention_heads = max(16, self.num_heads) + # Keep metadata sizing consistent with the padded tensor shape passed + # to the sparse decode kernel. + self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads( + self.num_heads + ) q_dtype = self.model_dtype kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")