[ROCm] Pad non-aligned AITER MLA heads (#51647)
Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
12f64b39d2
commit
5d8a4cf976
@@ -1,13 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Head-count padding + kernel selection for the ROCm AITER MLA backend.
|
||||
"""Head padding and kernel selection for the ROCm AITER MLA backend.
|
||||
|
||||
Kimi-K3 at TP8 puts 12 heads/rank and at TP16 puts 6 heads/rank on the AITER
|
||||
MLA decode. The asm persistent decode requires exactly 16 heads, so small
|
||||
head counts are tile-padded to 16 and the padding heads are sliced back off
|
||||
the output. Divisor counts (1/2/4/8) may keep the Gluon kernel, but only on
|
||||
gfx950 where that kernel has a build; every small head count on gfx942 (which
|
||||
has no Gluon build) is routed to the asm persistent decode instead.
|
||||
The asm persistent decode requires a 16-aligned head count, so unaligned
|
||||
counts through 128 are tile-padded to the next multiple of 16 and sliced back
|
||||
off the output. Small divisor counts (1/2/4/8) preserve their existing
|
||||
repeat-interleave path and may keep the Gluon kernel on gfx950.
|
||||
"""
|
||||
|
||||
import math
|
||||
@@ -19,6 +17,7 @@ import torch.nn.functional as F
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLADecodeMetadata,
|
||||
AiterMLAHelper,
|
||||
@@ -33,12 +32,20 @@ QK_HEAD_DIM = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
CONTEXT_LEN = 4096
|
||||
SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
|
||||
|
||||
# Non-divisor counts must go through the tile-and-slice path; divisor counts
|
||||
# (of 16) keep repeat_interleave. Both must pad to exactly 16 and round-trip.
|
||||
# Small non-divisor counts use tile-and-slice; divisors of 16 keep
|
||||
# repeat_interleave. Both pad to exactly 16 and round-trip.
|
||||
NON_DIVISOR_HEADS = [3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15]
|
||||
DIVISOR_HEADS = [1, 2, 4, 8]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _disable_native_h24(monkeypatch):
|
||||
"""Exercise the padding fallback unless a test explicitly enables H24."""
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_supported", lambda: False
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_available() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found() and torch.cuda.is_available()
|
||||
|
||||
@@ -123,6 +130,72 @@ def test_h12_output_discards_padding_heads():
|
||||
torch.testing.assert_close(unpadded_o, o[:, :12])
|
||||
|
||||
|
||||
def test_h24_query_is_tile_padded_to_h32():
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
|
||||
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(padded_q[:, :24], q)
|
||||
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
|
||||
|
||||
|
||||
def test_h24_output_discards_h32_padding_heads():
|
||||
o = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
|
||||
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(24, o)
|
||||
|
||||
assert unpadded_o.shape == (2, 24, 4)
|
||||
torch.testing.assert_close(unpadded_o, o[:, :24])
|
||||
|
||||
|
||||
def test_h24_reducer_without_metadata_still_pads_to_h32(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_native_h24_supported",
|
||||
lambda: (
|
||||
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
|
||||
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
|
||||
),
|
||||
)
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 32
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(24, q)
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
torch.testing.assert_close(padded_q[:, :24], q)
|
||||
torch.testing.assert_close(padded_q[:, 24:], q[:, :8])
|
||||
|
||||
|
||||
def test_native_h24_requires_reducer_and_metadata(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_reducer_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "_aiter_mla_native_h24_metadata_supported", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_aiter_mla_native_h24_supported",
|
||||
lambda: (
|
||||
rocm_aiter_mla._aiter_mla_native_h24_reducer_supported()
|
||||
and rocm_aiter_mla._aiter_mla_native_h24_metadata_supported()
|
||||
),
|
||||
)
|
||||
q = torch.arange(2 * 24 * 4, dtype=torch.float32).view(2, 24, 4)
|
||||
|
||||
assert AiterMLAHelper.get_actual_mla_num_heads(24) == 24
|
||||
assert AiterMLAHelper.get_mla_padded_q(24, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(24, q) is q
|
||||
|
||||
|
||||
def test_existing_divisor_head_mapping_is_unchanged():
|
||||
q = torch.arange(2 * 8 * 4, dtype=torch.bfloat16).view(2, 8, 4)
|
||||
|
||||
@@ -134,6 +207,18 @@ def test_existing_divisor_head_mapping_is_unchanged():
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [17, 24, 31])
|
||||
def test_unaligned_head_counts_round_trip_through_h32(num_heads: int):
|
||||
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
|
||||
|
||||
padded_q = AiterMLAHelper.get_mla_padded_q(num_heads, q)
|
||||
unpadded_o = AiterMLAHelper.get_mla_unpadded_o(num_heads, padded_q)
|
||||
|
||||
assert padded_q.shape == (2, 32, 4)
|
||||
assert padded_q.is_contiguous()
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", NON_DIVISOR_HEADS + DIVISOR_HEADS)
|
||||
def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
|
||||
q = torch.arange(2 * num_heads * 4, dtype=torch.float32).view(2, num_heads, 4)
|
||||
@@ -147,26 +232,30 @@ def test_all_small_head_counts_pad_to_16_and_round_trip(num_heads: int):
|
||||
torch.testing.assert_close(unpadded_o, q)
|
||||
|
||||
|
||||
def test_num_heads_ge_16_is_passthrough():
|
||||
q = torch.arange(2 * 16 * 4, dtype=torch.float32).view(2, 16, 4)
|
||||
assert AiterMLAHelper.get_mla_padded_q(16, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(16, q) is q
|
||||
def test_aligned_h32_is_zero_copy():
|
||||
q = torch.arange(2 * 32 * 4, dtype=torch.float32).view(2, 32, 4)
|
||||
assert AiterMLAHelper.get_mla_padded_q(32, q) is q
|
||||
assert AiterMLAHelper.get_mla_unpadded_o(32, q) is q
|
||||
|
||||
|
||||
def test_is_valid_num_heads():
|
||||
for n in range(1, 16):
|
||||
for n in range(1, 129):
|
||||
assert AiterMLAHelper.is_valid_num_heads(n)
|
||||
assert AiterMLAHelper.is_valid_num_heads(16)
|
||||
assert AiterMLAHelper.is_valid_num_heads(32)
|
||||
assert AiterMLAHelper.is_valid_num_heads(24)
|
||||
assert AiterMLAHelper.is_valid_num_heads(127)
|
||||
# Aligned counts remain valid above the range where padding is supported.
|
||||
assert AiterMLAHelper.is_valid_num_heads(144)
|
||||
assert not AiterMLAHelper.is_valid_num_heads(0)
|
||||
assert not AiterMLAHelper.is_valid_num_heads(129)
|
||||
|
||||
|
||||
def test_nondivisor_and_multitoken_never_use_gluon():
|
||||
# Non-divisor decode always takes the asm path (12 heads/rank at TP8).
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 1, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(6, 1, "auto")
|
||||
# >=16 heads never pad, never Gluon.
|
||||
# >=16 heads never use Gluon, including unaligned counts padded for asm.
|
||||
assert not AiterMLAHelper.use_gluon_decode(16, 1, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(24, 1, "auto")
|
||||
# Multi-token (verify / qlen>1) is never the single-token Gluon decode.
|
||||
assert not AiterMLAHelper.use_gluon_decode(8, 4, "auto")
|
||||
assert not AiterMLAHelper.use_gluon_decode(12, 4, "auto")
|
||||
|
||||
@@ -15,6 +15,7 @@ if not current_platform.is_rocm():
|
||||
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
|
||||
AiterMLAHelper,
|
||||
AiterMLAMetadataBuilder,
|
||||
)
|
||||
|
||||
@@ -83,7 +84,7 @@ def _builder(
|
||||
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
|
||||
_use_persistent_metadata=False,
|
||||
kernel_block_size=kernel_block_size,
|
||||
_num_attention_heads=16,
|
||||
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
|
||||
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
|
||||
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
|
||||
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
|
||||
@@ -109,7 +110,7 @@ def test_backend_declares_uniform_batch_support():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [8, 16, 32, 64, 128])
|
||||
@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128])
|
||||
@pytest.mark.parametrize(
|
||||
"spec_method, parallel_drafting",
|
||||
[
|
||||
@@ -127,8 +128,8 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
):
|
||||
"""Aiter init sizes the metadata for every query length decode can be handed.
|
||||
|
||||
Sweeping num_heads asserts the max(16, num_heads) clamp is what sizes the
|
||||
metadata, covering the fp8 nhead=32 (TP4) fold path.
|
||||
Sweeping num_heads asserts metadata is sized for the padded decode shape,
|
||||
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
|
||||
"""
|
||||
|
||||
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
|
||||
@@ -207,7 +208,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
{
|
||||
"max_batch_size": config.scheduler_config.max_num_seqs,
|
||||
"max_qo_len": builder.reorder_batch_threshold,
|
||||
"num_attention_heads": max(16, num_heads),
|
||||
"num_attention_heads": AiterMLAHelper.get_actual_mla_num_heads(num_heads),
|
||||
"q_dtype": dtypes.fp8,
|
||||
"kv_dtype": dtypes.fp8,
|
||||
"is_sparse": False,
|
||||
@@ -381,6 +382,7 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch):
|
||||
(1, 1, 16, "auto", True), # non-MTP decode
|
||||
(4, 2, 16, "auto", True), # MTP deployment, in-range step
|
||||
(4, 4, 16, "auto", True), # MTP deployment, full-qlen verification step
|
||||
(1, 1, 24, "auto", True), # unaligned H24 pads to H32 persistent decode
|
||||
(2, 4, 16, "auto", False), # step demand exceeds provisioned K -> fallback
|
||||
(1, 1, 8, "auto", False), # divisor head count -> Gluon decode owns qlen==1
|
||||
(4, 4, 8, "auto", False), # small head count -> Gluon flatten owns qlen>1
|
||||
@@ -410,7 +412,8 @@ def test_persistent_metadata_gate(
|
||||
K = _mtp_decode_qlen sizes the metadata buffers at init; a decode step gets
|
||||
the pre-built schedule only when its qlen fits those buffers, otherwise it
|
||||
falls back to the kernel computing its own. qlen==1 (non-MTP) must stay
|
||||
in-range -- dropping it is the regression this guards.
|
||||
in-range -- dropping it is the regression this guards. This includes
|
||||
unaligned H24, whose padded H32 decode uses persistent metadata.
|
||||
|
||||
Only the Gluon paths ignore the schedule, so the gate follows the routing
|
||||
predicates rather than the raw head count. Reading `num_heads >= 16` instead
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import functools
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import ClassVar, Final
|
||||
|
||||
import torch
|
||||
@@ -77,6 +78,48 @@ def _fp8_mla_prefill_supported() -> bool:
|
||||
return True
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _aiter_mla_native_h24_reducer_supported() -> bool:
|
||||
"""Whether AITER's JIT reducer supports the native H24/512 shape."""
|
||||
try:
|
||||
from aiter.jit.core import AITER_CSRC_DIR
|
||||
|
||||
reduce_source = Path(AITER_CSRC_DIR) / "kernels" / "mla" / "reduce.cu"
|
||||
source = "".join(reduce_source.read_text(encoding="utf-8").split())
|
||||
except (ImportError, OSError):
|
||||
return False
|
||||
return "MLA_REDUCE_CASE_EF(NUM_HEAD,24,HEAD_DIM,512," in source
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _aiter_mla_native_h24_metadata_supported() -> bool:
|
||||
"""Whether AITER's fast MLA metadata planner accepts native H24.
|
||||
|
||||
The reducer and metadata planner have independent shape dispatch. Checking
|
||||
only the reducer can route H24 into a planner that rejects it before the
|
||||
attention kernel launches. Until AITER exposes a capability API, inspect
|
||||
the shipped JIT source for an explicit native-H24 planner branch.
|
||||
"""
|
||||
try:
|
||||
from aiter.jit.core import AITER_CSRC_DIR
|
||||
|
||||
metadata_source = (
|
||||
Path(AITER_CSRC_DIR) / "kernels" / "mla" / "metadata" / "v1_2_device.cuh"
|
||||
)
|
||||
source = "".join(metadata_source.read_text(encoding="utf-8").split())
|
||||
except (ImportError, OSError):
|
||||
return False
|
||||
return "num_heads==24" in source
|
||||
|
||||
|
||||
def _aiter_mla_native_h24_supported() -> bool:
|
||||
"""Whether the complete AITER decode path supports native H24."""
|
||||
return (
|
||||
_aiter_mla_native_h24_reducer_supported()
|
||||
and _aiter_mla_native_h24_metadata_supported()
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _gluon_mla_decode_supported() -> bool:
|
||||
"""The small-head Gluon MLA decode kernel only has a gfx950 (CDNA4) build.
|
||||
@@ -308,10 +351,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
|
||||
from aiter import dtypes, get_mla_metadata_info_v1
|
||||
|
||||
# For num_attention_heads < 16 (e.g. kimi-k2.5 head=8 with TP8),
|
||||
# make sure get_mla_metadata_info_v1 / get_mla_metadata_v1 are consistent
|
||||
# with the actual tensor shape passed to mla_decode_fwd.
|
||||
self._num_attention_heads = max(16, self.num_heads)
|
||||
# Keep metadata sizing consistent with the padded tensor shape passed
|
||||
# to mla_decode_fwd.
|
||||
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
|
||||
self.num_heads
|
||||
)
|
||||
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
|
||||
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
|
||||
kv_cache_dtype_str = "fp8"
|
||||
@@ -839,14 +883,16 @@ def _expand_page_indices_kernel(
|
||||
|
||||
class AiterMLAHelper:
|
||||
"""
|
||||
AITER MLA persistent (asm) decode requires num_heads >= 16. Head counts
|
||||
< 16 are padded up to exactly 16: divisors of 16 by repeat_interleave,
|
||||
other counts (e.g. 12 heads/rank at TP8, 6 at TP16) by tiling the query
|
||||
heads and slicing to 16. Non-divisor padded decodes take the asm path;
|
||||
divisors and max_qo_len > 1 small-head verify still use Gluon.
|
||||
AITER MLA persistent (asm) decode requires a multiple of 16 heads. Unaligned
|
||||
head counts through 128 are padded to the next multiple of 16 by tiling the
|
||||
query heads and slicing to the padded size. Native H24 AITER builds bypass
|
||||
that padding. Small divisors of 16 retain the existing repeat_interleave and
|
||||
strided-unpad behavior. Native and aligned counts pass through without
|
||||
copies.
|
||||
"""
|
||||
|
||||
_AITER_MIN_MLA_HEADS: Final = 16
|
||||
_AITER_MAX_PADDED_MLA_HEADS: Final = 128
|
||||
# Largest qlen the padded gqa=16 asm decode has a bf16 persistent kernel
|
||||
# for. Above it only the non-persistent qseqlen=8 entry exists, and the
|
||||
# fold that reaches a persistent one is gfx950-only.
|
||||
@@ -856,48 +902,50 @@ class AiterMLAHelper:
|
||||
@staticmethod
|
||||
def check_num_heads_validity(num_heads: int):
|
||||
assert AiterMLAHelper.is_valid_num_heads(num_heads), (
|
||||
"ROCM AITER MLA requires 1-15 heads (padded to 16 for asm "
|
||||
"persistent decode; exact divisors of 16 may keep Gluon) or a "
|
||||
f"multiple of 16 heads, but got {num_heads}.\n"
|
||||
"ROCM AITER MLA requires a positive multiple of 16 heads, or an "
|
||||
"unaligned head count up to 128 (padded to the next multiple of "
|
||||
f"16), but got {num_heads}.\n"
|
||||
f"Try adjusting tensor_parallel_size value."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_valid_num_heads(num_heads: int) -> bool:
|
||||
return num_heads > 0 and (
|
||||
num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS
|
||||
or num_heads % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
|
||||
return (
|
||||
num_heads > 0
|
||||
and num_heads not in AiterMLAHelper._AITER_UNSUPPORTED_HEADS
|
||||
and (
|
||||
num_heads <= AiterMLAHelper._AITER_MAX_PADDED_MLA_HEADS
|
||||
or num_heads % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_actual_mla_num_heads(num_heads: int) -> int:
|
||||
return max(num_heads, AiterMLAHelper._AITER_MIN_MLA_HEADS)
|
||||
if num_heads == 24 and _aiter_mla_native_h24_supported():
|
||||
return num_heads
|
||||
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
|
||||
return -(-num_heads // m) * m
|
||||
|
||||
@staticmethod
|
||||
def get_mla_padded_q(num_heads: int, q: torch.Tensor) -> torch.Tensor:
|
||||
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
|
||||
if num_heads >= m:
|
||||
m = AiterMLAHelper.get_actual_mla_num_heads(num_heads)
|
||||
if num_heads == m:
|
||||
return q
|
||||
if m % num_heads == 0:
|
||||
return q.repeat_interleave(m // num_heads, dim=1)
|
||||
# Non-divisor head counts (e.g. 12 heads/rank at TP8, 6 at TP16) cannot
|
||||
# be padded by repeat_interleave. Tile the query heads and slice to
|
||||
# exactly m; this reaches m for any 0 < num_heads < m (unlike a single
|
||||
# append, which under-pads when num_heads < m - num_heads). MLA
|
||||
# attention is independent per query head over the shared KV, so the
|
||||
# padding heads cannot affect heads [0:num_heads]; they are sliced back
|
||||
# off in get_mla_unpadded_o.
|
||||
# Non-divisor head counts cannot be padded by repeat_interleave. Tile
|
||||
# the query heads and slice to exactly m. MLA attention is independent
|
||||
# per query head over the shared KV, so padding heads cannot affect
|
||||
# heads [0:num_heads]; they are sliced back off the output.
|
||||
reps = -(-m // num_heads) # ceil(m / num_heads)
|
||||
# Slicing a tiled tensor down to m yields a non-contiguous view whenever
|
||||
# reps * num_heads > m (the common case: TP8 12->24->16, TP16 6->18->16).
|
||||
# The asm persistent decode reads q as a packed [tokens, m, head_dim]
|
||||
# buffer, so materialize a contiguous copy. No-op when already contiguous.
|
||||
# Slicing a tiled tensor yields a non-contiguous view. The asm decode
|
||||
# reads q as packed [tokens, m, head_dim], so materialize it.
|
||||
return q.repeat(1, reps, 1)[:, :m, :].contiguous()
|
||||
|
||||
@staticmethod
|
||||
def get_mla_unpadded_o(num_heads: int, o: torch.Tensor) -> torch.Tensor:
|
||||
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
|
||||
if num_heads >= m:
|
||||
m = AiterMLAHelper.get_actual_mla_num_heads(num_heads)
|
||||
if num_heads == m:
|
||||
return o
|
||||
if m % num_heads == 0:
|
||||
return o[:, :: m // num_heads, :]
|
||||
|
||||
@@ -422,9 +422,11 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
# so the buffers are large enough for any decode shape we might see.
|
||||
from aiter import dtypes, get_mla_metadata_info_v1
|
||||
|
||||
# Aiter sparse MLA also requires num_heads >= 16 (will be padded by
|
||||
# AiterMLAHelper.get_mla_padded_q in forward).
|
||||
self._num_attention_heads = max(16, self.num_heads)
|
||||
# Keep metadata sizing consistent with the padded tensor shape passed
|
||||
# to the sparse decode kernel.
|
||||
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
|
||||
self.num_heads
|
||||
)
|
||||
|
||||
q_dtype = self.model_dtype
|
||||
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
|
||||
|
||||
Reference in New Issue
Block a user