[ROCm] Pad non-aligned AITER MLA heads (#51647)
Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
12f64b39d2
commit
5d8a4cf976
@@ -1,13 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Head-count padding + kernel selection for the ROCm AITER MLA backend.
|
||||
"""Head padding and kernel selection for the ROCm AITER MLA backend.
|
||||
|
||||
Kimi-K3 at TP8 puts 12 heads/rank and at TP16 puts 6 heads/rank on the AITER
|
||||
MLA decode. The asm persistent decode requires exactly 16 heads, so small
|
||||
head counts are tile-padded to 16 and the padding heads are sliced back off
|
||||
the output. Divisor counts (1/2/4/8) may keep the Gluon kernel, but only on
|
||||
gfx950 where that kernel has a build; every small head count on gfx942 (which
|
||||
has no Gluon build) is routed to the asm persistent decode instead.
|
||||
The asm persistent decode requires a 16-aligned head count, so unaligned
|
||||
counts through 128 are tile-padded to the next multiple of 16 and sliced back
|
||||
off the output. Small divisor counts (1/2/4/8) preserve their existing
|
||||
repeat-interleave path and may keep the Gluon kernel on gfx950.
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -19,6 +17,7 @@ import torch.nn.functional as F
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLADecodeMetadata,
|
||||
AiterMLAHelper,
|
||||
@@ -33,12 +32,20 @@ QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
CONTEXT_LEN = 4096
|
||||
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
|
||||
|
||||
# Non-divisor counts must go through the tile-and-slice path; divisor counts
|
||||
# (of 16) keep repeat_interleave. Both must pad to exactly 16 and round-trip.
|
||||
# Small non-divisor counts use tile-and-slice; divisors of 16 keep
|
||||
# repeat_interleave. Both pad to exactly 16 and round-trip.
|
||||
NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15]
|
||||
DIVISOR_HEADS = [1, 2, 4, 8]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _disable_native_h24(monkeypatch):
|
||||
"""Exercise the padding fallback unless a test explicitly enables H24."""
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_available() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available()
|
||||
|
||||
@@ -123,6 +130,72 @@ def test_h12_output_discards_padding_heads():
|
||||
torch.testing.assert_close(unpadded_o, o[:, :12])
|
||||
|
||||
|
||||
def test_h24_query_is_tile_padded_to_h32():
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
|
||||
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(padded_q[:, :24], q)
|
||||
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
|
||||
|
||||
|
||||
def test_h24_output_discards_h32_padding_heads():
|
||||
o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
|
||||
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o)
|
||||
|
||||
assert unpadded_o.shape == (2, 24, 4)
|
||||
torch.testing.assert_close(unpadded_o, o[:, :24])
|
||||
|
||||
|
||||
def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_native_h24_supported",
|
||||
lambda: (
|
||||
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
|
||||
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
|
||||
),
|
||||
)
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
torch.testing.assert_close(padded_q[:, :24], q)
|
||||
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
|
||||
|
||||
|
||||
def test_native_h24_requires_reducer_and_metadata(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_native_h24_supported",
|
||||
lambda: (
|
||||
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
|
||||
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
|
||||
),
|
||||
)
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24
|
||||
assert AiterMLAHelper.get_mla_padded_q(24, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q
|
||||
|
||||
|
||||
def test_existing_divisor_head_mapping_is_unchanged():
|
||||
q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4)
|
||||
|
||||
@@ -134,6 +207,18 @@ def test_existing_divisor_head_mapping_is_unchanged():
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [17, 24, 31])
|
||||
def test_unaligned_head_counts_round_trip_through_h32(num_heads: int):
|
||||
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
|
||||
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS)
|
||||
def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
|
||||
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
|
||||
@@ -147,26 +232,30 @@ def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
def test_num_heads_ge_16_is_passthrough():
|
||||
q = torch.arange(2 * 16 * 4, dtype=torch.float32).view(2, 16, 4)
|
||||
assert AiterMLAHelper.get_mla_padded_q(16, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(16, q) is q
|
||||
def test_aligned_h32_is_zero_copy():
|
||||
q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
|
||||
assert AiterMLAHelper.get_mla_padded_q(32, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q
|
||||
|
||||
|
||||
def test_is_valid_num_heads():
|
||||
for n in range(1, 16):
|
||||
for n in range(1, 129):
|
||||
assert AiterMLAHelper.is_valid_num_heads(n)
|
||||
assert AiterMLAHelper.is_valid_num_heads(16)
|
||||
assert AiterMLAHelper.is_valid_num_heads(32)
|
||||
assert AiterMLAHelper.is_valid_num_heads(24)
|
||||
assert AiterMLAHelper.is_valid_num_heads(127)
|
||||
# Aligned counts remain valid above the range where padding is supported.
|
||||
assert AiterMLAHelper.is_valid_num_heads(144)
|
||||
assert not AiterMLAHelper.is_valid_num_heads(0)
|
||||
assert not AiterMLAHelper.is_valid_num_heads(129)
|
||||
|
||||
|
||||
def test_nondivisor_and_multitoken_never_use_gluon():
|
||||
# Non-divisor decode always takes the asm path (12 heads/rank at TP8).
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto")
|
||||
# >=16 heads never pad, never Gluon.
|
||||
# >=16 heads never use Gluon, including unaligned counts padded for asm.
|
||||
assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto")
|
||||
# Multi-token (verify / qlen>1) is never the single-token Gluon decode.
|
||||
assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto")
|
||||
|
||||
@@ -15,6 +15,7 @@ if not current_platform.is_rocm():
|
||||
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
|
||||
AiterMLAHelper,
|
||||
AiterMLAMetadataBuilder,
|
||||
)
|
||||
|
||||
@@ -83,7 +84,7 @@ def _builder(
|
||||
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
|
||||
_use_persistent_metadata=False,
|
||||
kernel_block_size=kernel_block_size,
|
||||
_num_attention_heads=16,
|
||||
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
|
||||
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
|
||||
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
|
||||
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
|
||||
@@ -109,7 +110,7 @@ def test_backend_declares_uniform_batch_support():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [8, 16, 32, 64, 128])
|
||||
@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128])
|
||||
@pytest.mark.parametrize(
|
||||
"spec_method, parallel_drafting",
|
||||
[
|
||||
@@ -127,8 +128,8 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
):
|
||||
"""Aiter init sizes the metadata for every query length decode can be handed.
|
||||
|
||||
Sweeping num_heads asserts the max(16, num_heads) clamp is what sizes the
|
||||
metadata, covering the fp8 nhead=32 (TP4) fold path.
|
||||
Sweeping num_heads asserts metadata is sized for the padded decode shape,
|
||||
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
|
||||
"""
|
||||
|
||||
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
|
||||
@@ -207,7 +208,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
{
|
||||
"max_batch_size": config.scheduler_config.max_num_seqs,
|
||||
"max_qo_len": builder.reorder_batch_threshold,
|
||||
"num_attention_heads": max(16, num_heads),
|
||||
"num_attention_heads": AiterMLAHelper.get_actual_mla_num_heads(num_heads),
|
||||
"q_dtype": dtypes.fp8,
|
||||
"kv_dtype": dtypes.fp8,
|
||||
"is_sparse": False,
|
||||
@@ -381,6 +382,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch):
|
||||
(1, 1, 16, "auto", True), # non-MTP decode
|
||||
(4, 2, 16, "auto", True), # MTP deployment, in-range step
|
||||
(4, 4, 16, "auto", True), # MTP deployment, full-qlen verification step
|
||||
(1, 1, 24, "auto", True), # unaligned H24 pads to H32 persistent decode
|
||||
(2, 4, 16, "auto", False), # step demand exceeds provisioned K -> fallback
|
||||
(1, 1, 8, "auto", False), # divisor head count -> Gluon decode owns qlen==1
|
||||
(4, 4, 8, "auto", False), # small head count -> Gluon flatten owns qlen>1
|
||||
@@ -410,7 +412,8 @@ def test_persistent_metadata_gate(
|
||||
K = _mtp_decode_qlen sizes the metadata buffers at init; a decode step gets
|
||||
the pre-built schedule only when its qlen fits those buffers, otherwise it
|
||||
falls back to the kernel computing its own. qlen==1 (non-MTP) must stay
|
||||
in-range -- dropping it is the regression this guards.
|
||||
in-range -- dropping it is the regression this guards. This includes
|
||||
unaligned H24, whose padded H32 decode uses persistent metadata.
|
||||
|
||||
Only the Gluon paths ignore the schedule, so the gate follows the routing
|
||||
predicates rather than the raw head count. Reading `num_heads >= 16` instead
|
||||
|
||||
Reference in New Issue
Block a user