[ROCm][K3] Extend FP8 asm MLA prefill to non-divisor small head counts (#51040)

Signed-off-by: Xiaohu Guo <Xiaohu.Guo@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
xiaohuguo2023
2026-08-26 17:47:16 +00:00
committed by GitHub
co-authored by Cursor
parent 17da48596c
commit 1a085dadf2
2 changed files with 248 additions and 44 deletions
@@ -43,8 +43,20 @@ pytestmark = pytest.mark.skipif(
)
# DeepSeek-style MLA head dims after kv_b_proj decompression (q/k carry the
# nope+rope dims, v carries v_head_dim). FP8 prefill requires 16-aligned heads.
NUM_HEADS = 16
# nope+rope dims, v carries v_head_dim).
#
# The kernel requires 16-aligned heads, so _mla_fp8_prefill_attn replicate-pads
# q/k/v up to get_fp8_prefill_num_heads(num_heads) and slices the output back.
# Cover all three regimes:
# 12 -> padded to 16 (non-divisor, tile+slice). Live case: a K3 rank at TP8.
# 16 -> no padding, output aliases the caller's buffer.
# 24 -> padded to 32. No current model and TP reaches this band (96 heads
# would need TP4, which does not fit in 288 GiB; 128-head models give
# 128/64/32/16/8), so this case exists to keep the general path honest
# for future architectures: it is the only one of the three where
# padding above 16 actually happens, so it is what proves the
# replicate-pad/slice-back argument holds for a target other than 16.
HEAD_COUNTS = [12, 16, 24]
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
QK_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM # 192
@@ -56,6 +68,10 @@ SCALE = 1.0 / math.sqrt(QK_HEAD_DIM)
# elements.
ATOL, RTOL = 1e-1, 5e-2
# The assembly kernel writes bf16 through a raw output pointer. Keep its tensors
# and workspace reservation on one dtype.
ATTN_OUT_DTYPE = torch.bfloat16
@pytest.fixture(autouse=True)
def _workspace_manager():
@@ -73,14 +89,14 @@ def _workspace_manager():
reset_workspace_manager()
def _make_impl():
def _make_impl(num_heads: int):
"""Minimal AiterMLAImpl exposing only what _mla_fp8_prefill_attn reads."""
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.num_heads = NUM_HEADS
impl.num_heads = num_heads
impl.v_head_dim = V_HEAD_DIM
impl.scale = SCALE
impl._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
@@ -88,8 +104,13 @@ def _make_impl():
return impl
def _build_prefill_metadata(seq_lens: list[int], device: torch.device):
"""Build fp8_prefill_* metadata via the real builder (no hand-rolling)."""
def _build_prefill_metadata(
seq_lens: list[int],
device: torch.device,
num_heads: int,
attn_out_dtype: torch.dtype = ATTN_OUT_DTYPE,
):
"""Build metadata with the production builder and output dtype."""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
qo_indptr_cpu = torch.zeros(len(seq_lens) + 1, dtype=torch.int32)
@@ -99,12 +120,13 @@ def _build_prefill_metadata(seq_lens: list[int], device: torch.device):
max_q = max(seq_lens)
builder = object.__new__(AiterMLAMetadataBuilder)
builder.num_heads = NUM_HEADS
builder.num_heads = num_heads
builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM)
builder._init_fp8_prefill_ps_buffers(
max_num_reqs=len(seq_lens),
max_prefill_qlen=max_q,
max_num_batched_tokens=total_q,
attn_out_dtype=attn_out_dtype,
device=device,
)
@@ -115,7 +137,7 @@ def _build_prefill_metadata(seq_lens: list[int], device: torch.device):
metadata = SimpleNamespace(prefill=prefill, num_decodes=0)
common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu)
builder._build_fp8_prefill_ps_metadata(metadata, common)
return metadata, total_q
return metadata, total_q, builder
def _reference(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
@@ -128,34 +150,95 @@ def _reference(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tenso
return out.transpose(0, 1) # [S, H, Dv]
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
@pytest.mark.parametrize("seq_len", [128, 512])
@torch.inference_mode()
def test_fp8_prefill_matches_reference(seq_len: int) -> None:
def test_fp8_prefill_matches_reference(seq_len: int, num_heads: int) -> None:
device = torch.device("cuda")
torch.manual_seed(0)
metadata, total_q = _build_prefill_metadata([seq_len], device)
metadata, total_q, _ = _build_prefill_metadata([seq_len], device, num_heads)
q = torch.randn(
total_q, NUM_HEADS, QK_HEAD_DIM, dtype=torch.bfloat16, device=device
)
k = torch.randn(
total_q, NUM_HEADS, QK_HEAD_DIM, dtype=torch.bfloat16, device=device
)
v = torch.randn(total_q, NUM_HEADS, V_HEAD_DIM, dtype=torch.bfloat16, device=device)
# Lock the workspace, as GPUModelRunner does after warmup, so the forward
# cannot quietly grow it. The builder reserves scratch at num_head_k and
# the forward requests it at its own padded width; if a future change makes
# the forward's width exceed the builder's, that is an under-reservation
# which is invisible on an unlocked workspace (it just grows) and fatal in
# a real serve. Locking turns it into a failure here.
#
# Note this does not catch the reverse -- a builder wider than the forward
# merely over-reserves -- so it would not have flagged the pre-PR
# max(16, num_heads) sizing, which over-reserves 2.8x at 24 heads.
from vllm.v1.worker.workspace import lock_workspace
lock_workspace()
qkv_kwargs = dict(dtype=ATTN_OUT_DTYPE, device=device)
q = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
k = torch.randn(total_q, num_heads, QK_HEAD_DIM, **qkv_kwargs)
v = torch.randn(total_q, num_heads, V_HEAD_DIM, **qkv_kwargs)
out = torch.zeros(
total_q, NUM_HEADS * V_HEAD_DIM, dtype=torch.bfloat16, device=device
total_q, num_heads * V_HEAD_DIM, dtype=ATTN_OUT_DTYPE, device=device
)
impl = _make_impl()
impl = _make_impl(num_heads)
impl._mla_fp8_prefill_attn(q, k, v, metadata, out)
out_ref = _reference(q, k, v)
assert torch.isfinite(out).all()
torch.testing.assert_close(
out.view(total_q, NUM_HEADS, V_HEAD_DIM).float(),
out.view(total_q, num_heads, V_HEAD_DIM).float(),
out_ref.float(),
atol=ATOL,
rtol=RTOL,
)
@pytest.mark.parametrize("num_heads", HEAD_COUNTS)
def test_fp8_prefill_metadata_width_matches_forward(num_heads: int) -> None:
"""The PS metadata must be built for the head count the kernel is handed.
``_init_fp8_prefill_ps_buffers``/``build()`` size the work and reduce maps
from ``num_head_k``, while ``_mla_fp8_prefill_attn`` pads q/k/v to its own
target. Divergence is not a numerical bug in either direction -- maps built
narrower than the kernel width still cover the real heads, since padding
only ever appends duplicate heads above them. It is a sizing bug: narrower
maps mean a lower head alignment, hence more partial tiles and a larger
reservation, while wider ones under-reserve the forward's scratch. Pin the
two to one helper rather than to independent expressions.
"""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAHelper
width = AiterMLAHelper.get_fp8_prefill_num_heads(num_heads)
assert width % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
assert width >= num_heads
assert width - num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS
# Every count the enable gate accepts must have a usable padded width.
assert AiterMLAHelper.is_valid_num_heads(num_heads)
# The builder must actually size its maps at that width. Assert against the
# buffer it allocated rather than recomputing the width here, or the test
# passes no matter what _init_fp8_prefill_ps_buffers does.
#
# fp8_ps_reduce_partial_map is sized by the partial-tile count, which is
# gcd-driven and so is a sharp probe of the width used: a lower head
# alignment yields *more* tiles. At 24 heads, sizing at the unpadded count
# gives 4x the tiles (64 vs 16) and ~2.8x the scratch reservation (193.6
# vs 68.6 MiB at batch=1/qlen=512) -- not wrong numerically (the 24-wide
# maps still cover the real heads) but the reason to pin the widths
# together.
from aiter import get_ps_metadata_info_v1
from vllm.v1.attention.backends.mla.rocm_aiter_mla import _FP8_PREFILL_TILE_Q
def _tiles(nhk: int) -> int:
return get_ps_metadata_info_v1(
batch_size=1,
num_head_k=nhk,
max_qlen=512,
qlen_granularity=_FP8_PREFILL_TILE_Q,
)[5][0]
_, _, builder = _build_prefill_metadata([512], torch.device("cuda"), num_heads)
assert builder.fp8_ps_reduce_partial_map.numel() == _tiles(width)