[ROCm][MLA] FP8 ASM prefill for AITER dense MLA backend on gfx950 (#42509)
Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com> Co-authored-by: clintg6 <clint.greene@amd.com> Co-authored-by: frida-andersson <frida.andersson@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
clintg6
frida-andersson
Cursor
parent
fb5bd03f51
commit
46a95815d3
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar, Final
|
||||
|
||||
@@ -9,6 +10,7 @@ import torch
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBackend,
|
||||
MLACommonDecodeMetadata,
|
||||
@@ -26,6 +28,28 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _fp8_mla_prefill_supported() -> bool:
|
||||
"""Auto-detect FP8 MLA prefill via mla_prefill_ps_asm_fwd + mla_reduce_v1.
|
||||
|
||||
Requires gfx950 plus an AITER build that exports both kernels. When
|
||||
either is missing we silently fall back to ``flash_attn_varlen_func``.
|
||||
"""
|
||||
try:
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
if not on_gfx950():
|
||||
return False
|
||||
try:
|
||||
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1 # noqa: F401
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class AiterMLABackend(MLACommonBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
@@ -91,6 +115,25 @@ class AiterMLAMetadata(MLACommonMetadata[AiterMLADecodeMetadata]):
|
||||
reduce_final_map: torch.Tensor | None = None
|
||||
reduce_partial_map: torch.Tensor | None = None
|
||||
|
||||
# FP8 ASM prefill persistent-scheduling (PS) metadata. Populated by
|
||||
# AiterMLAMetadataBuilder._build_fp8_prefill_ps_metadata when prefill
|
||||
# tokens are present and FP8 MLA prefill is supported on the device.
|
||||
# Left as None on hosts/configs that fall back to flash_attn_varlen_func.
|
||||
fp8_prefill_qo_indptr: torch.Tensor | None = None
|
||||
fp8_prefill_kv_indptr: torch.Tensor | None = None
|
||||
fp8_prefill_kv_indices: torch.Tensor | None = None
|
||||
fp8_prefill_work_indptr: torch.Tensor | None = None
|
||||
fp8_prefill_work_info_set: torch.Tensor | None = None
|
||||
fp8_prefill_reduce_indptr: torch.Tensor | None = None
|
||||
fp8_prefill_reduce_final_map: torch.Tensor | None = None
|
||||
fp8_prefill_reduce_partial_map: torch.Tensor | None = None
|
||||
fp8_prefill_max_q_len: int | None = None
|
||||
fp8_prefill_num_partial_tiles: int | None = None
|
||||
|
||||
|
||||
# Tile size used by the mla_prefill_ps_asm_fwd assembly kernel.
|
||||
_FP8_PREFILL_TILE_Q = 256
|
||||
|
||||
|
||||
class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# TODO(luka, lucas): audit this as part of:
|
||||
@@ -193,6 +236,14 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
device=device,
|
||||
)
|
||||
|
||||
self._fp8_prefill_enabled = _fp8_mla_prefill_supported()
|
||||
if self._fp8_prefill_enabled:
|
||||
max_prefill_qlen = min(
|
||||
vllm_config.model_config.max_model_len,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
self._init_fp8_prefill_ps_buffers(max_num_reqs, max_prefill_qlen, device)
|
||||
|
||||
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
|
||||
self.paged_kv_indptr = torch.zeros(
|
||||
max_num_reqs + 1, dtype=torch.int32, device=device
|
||||
@@ -202,6 +253,162 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
max_num_reqs + 1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
def _init_fp8_prefill_ps_buffers(
|
||||
self,
|
||||
max_num_reqs: int,
|
||||
max_prefill_qlen: int,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""Pre-allocate persistent buffers for FP8 MLA prefill PS metadata.
|
||||
|
||||
Uses ``get_ps_metadata_info_v1`` with max values so the buffers are
|
||||
large enough for any batch. ``get_ps_metadata_v1`` fills them
|
||||
per-batch in ``build()``.
|
||||
|
||||
Args:
|
||||
max_num_reqs: Maximum number of concurrent requests.
|
||||
max_prefill_qlen: Maximum Q-length for a single request in one
|
||||
prefill batch. Should be ``min(max_model_len,
|
||||
max_num_batched_tokens)`` — the chunked-prefill scheduler
|
||||
never emits more than ``max_num_batched_tokens`` new tokens
|
||||
per batch.
|
||||
device: Target device for the buffers.
|
||||
"""
|
||||
from aiter import get_ps_metadata_info_v1
|
||||
|
||||
# After kv_b_proj decompression, K has num_heads heads (same as Q).
|
||||
# So gqa_ratio=1 and num_head_k=num_heads for the PS kernel.
|
||||
num_head_k = self.num_heads
|
||||
# gqa_ratio = 1
|
||||
# qlen_granularity = _FP8_PREFILL_TILE_Q // max(gqa_ratio, 1)
|
||||
qlen_granularity = _FP8_PREFILL_TILE_Q
|
||||
|
||||
(
|
||||
(work_metadata_size, work_metadata_dtype),
|
||||
(work_indptr_size, work_indptr_dtype),
|
||||
(work_info_size, work_info_dtype),
|
||||
(reduce_indptr_size, reduce_indptr_dtype),
|
||||
(reduce_final_map_size, reduce_final_map_dtype),
|
||||
(reduce_partial_map_size, reduce_partial_map_dtype),
|
||||
) = get_ps_metadata_info_v1(
|
||||
batch_size=max_num_reqs,
|
||||
num_head_k=num_head_k,
|
||||
max_qlen=max_prefill_qlen,
|
||||
qlen_granularity=qlen_granularity,
|
||||
)
|
||||
|
||||
self.fp8_ps_work_metadata = torch.empty(
|
||||
work_metadata_size, dtype=work_metadata_dtype, device=device
|
||||
)
|
||||
self.fp8_ps_work_indptr = torch.empty(
|
||||
work_indptr_size, dtype=work_indptr_dtype, device=device
|
||||
)
|
||||
self.fp8_ps_work_info = torch.empty(
|
||||
*work_info_size, dtype=work_info_dtype, device=device
|
||||
)
|
||||
self.fp8_ps_reduce_indptr = torch.empty(
|
||||
reduce_indptr_size, dtype=reduce_indptr_dtype, device=device
|
||||
)
|
||||
self.fp8_ps_reduce_final_map = torch.empty(
|
||||
*reduce_final_map_size, dtype=reduce_final_map_dtype, device=device
|
||||
)
|
||||
self.fp8_ps_reduce_partial_map = torch.empty(
|
||||
reduce_partial_map_size,
|
||||
dtype=reduce_partial_map_dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"FP8 MLA prefill PS buffers allocated "
|
||||
"(max_batch=%d, max_qlen=%d, num_head_k=%d)",
|
||||
max_num_reqs,
|
||||
max_prefill_qlen,
|
||||
num_head_k,
|
||||
)
|
||||
|
||||
def _build_fp8_prefill_ps_metadata(
|
||||
self,
|
||||
metadata: AiterMLAMetadata,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> None:
|
||||
"""Build per-batch FP8 MLA prefill PS metadata and attach to *metadata*.
|
||||
|
||||
Called from ``build()`` when prefill tokens are present and
|
||||
FP8 MLA prefill is enabled (auto-detected via
|
||||
``_fp8_mla_prefill_supported()``).
|
||||
"""
|
||||
from aiter import get_ps_metadata_v1
|
||||
|
||||
prefill = metadata.prefill
|
||||
# Caller (build()) only invokes this when prefill tokens exist, so
|
||||
# metadata.prefill is guaranteed non-None. Assert to narrow for mypy.
|
||||
assert prefill is not None
|
||||
qo_indptr = prefill.query_start_loc
|
||||
kv_indptr = qo_indptr # new tokens: KV length == Q length
|
||||
|
||||
# Reuse the existing CPU view of query_start_loc instead of forcing a
|
||||
# device->host copy. Prefill batches sit at the tail of the request
|
||||
# list, so we slice from num_decodes onwards and rebase to zero, the
|
||||
# same transform the parent build applies on device tensors.
|
||||
num_decodes = metadata.num_decodes
|
||||
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
qo_indptr_cpu = (qsl_cpu[num_decodes:] - qsl_cpu[num_decodes]).to(torch.int32)
|
||||
kv_indptr_cpu = qo_indptr_cpu.clone()
|
||||
seq_lens_cpu = (qo_indptr_cpu[1:] - qo_indptr_cpu[:-1]).to(torch.int32)
|
||||
|
||||
num_head_k = self.num_heads
|
||||
# gqa_ratio = 1
|
||||
# qhead_granularity = max(gqa_ratio, 1)
|
||||
# qlen_granularity = _FP8_PREFILL_TILE_Q // qhead_granularity
|
||||
gqa_ratio = 1
|
||||
qhead_granularity = 1
|
||||
qlen_granularity = _FP8_PREFILL_TILE_Q
|
||||
kvlen_granularity = 128
|
||||
block_size = 1 # non-paged: each "page" is one token
|
||||
|
||||
get_ps_metadata_v1(
|
||||
qo_indptr_cpu,
|
||||
kv_indptr_cpu,
|
||||
seq_lens_cpu,
|
||||
gqa_ratio,
|
||||
num_head_k,
|
||||
self.fp8_ps_work_metadata,
|
||||
self.fp8_ps_work_indptr,
|
||||
self.fp8_ps_work_info,
|
||||
self.fp8_ps_reduce_indptr,
|
||||
self.fp8_ps_reduce_final_map,
|
||||
self.fp8_ps_reduce_partial_map,
|
||||
qhead_granularity=qhead_granularity,
|
||||
qlen_granularity=qlen_granularity,
|
||||
kvlen_granularity=kvlen_granularity,
|
||||
block_size=block_size,
|
||||
is_causal=True,
|
||||
)
|
||||
|
||||
total_prefill_tokens = int(qo_indptr_cpu[-1].item())
|
||||
kv_indices = torch.arange(
|
||||
total_prefill_tokens, device=qo_indptr.device, dtype=torch.int32
|
||||
)
|
||||
|
||||
# The actual number of active partial tiles for this batch is the
|
||||
# final value of reduce_indptr. Resolving it here (during metadata
|
||||
# build) keeps it off the per-layer forward path where a sync would
|
||||
# break CUDA Graph capture. Using the device-side reduce_indptr is
|
||||
# acceptable since build is allowed to incur an occasional sync.
|
||||
num_partial_tiles = int(self.fp8_ps_reduce_indptr[-1].item())
|
||||
|
||||
# Attach PS metadata to the metadata object so forward_mha can read it.
|
||||
metadata.fp8_prefill_qo_indptr = qo_indptr
|
||||
metadata.fp8_prefill_kv_indptr = kv_indptr
|
||||
metadata.fp8_prefill_kv_indices = kv_indices
|
||||
metadata.fp8_prefill_work_indptr = self.fp8_ps_work_indptr
|
||||
metadata.fp8_prefill_work_info_set = self.fp8_ps_work_info
|
||||
metadata.fp8_prefill_reduce_indptr = self.fp8_ps_reduce_indptr
|
||||
metadata.fp8_prefill_reduce_final_map = self.fp8_ps_reduce_final_map
|
||||
metadata.fp8_prefill_reduce_partial_map = self.fp8_ps_reduce_partial_map
|
||||
metadata.fp8_prefill_max_q_len = prefill.max_query_len
|
||||
metadata.fp8_prefill_num_partial_tiles = num_partial_tiles
|
||||
|
||||
def _build_decode(
|
||||
self,
|
||||
block_table_tensor: torch.Tensor,
|
||||
@@ -335,6 +542,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
attn_metadata.reduce_indptr = self._mla_reduce_indptr
|
||||
attn_metadata.reduce_final_map = self._mla_reduce_final_map
|
||||
attn_metadata.reduce_partial_map = self._mla_reduce_partial_map
|
||||
if self._fp8_prefill_enabled and attn_metadata.prefill is not None:
|
||||
self._build_fp8_prefill_ps_metadata(attn_metadata, common_attn_metadata)
|
||||
return attn_metadata
|
||||
|
||||
|
||||
@@ -479,6 +688,15 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
|
||||
self.flash_attn_varlen_func = flash_attn_varlen_func
|
||||
|
||||
# FP8 MLA prefill kernel imports (lazy, only when enabled).
|
||||
# Auto-enabled on gfx950 when AITER ships the kernels.
|
||||
self._fp8_prefill_enabled = _fp8_mla_prefill_supported()
|
||||
if self._fp8_prefill_enabled:
|
||||
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1
|
||||
|
||||
self._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
|
||||
self._mla_reduce_v1 = mla_reduce_v1
|
||||
|
||||
def _flash_attn_varlen_diff_headdims(
|
||||
self, q, k, v, return_softmax_lse=False, softmax_scale=None, **kwargs
|
||||
):
|
||||
@@ -493,6 +711,157 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
|
||||
return output
|
||||
|
||||
def _mla_fp8_prefill_attn(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
attn_metadata: AiterMLAMetadata,
|
||||
out: torch.Tensor,
|
||||
) -> None:
|
||||
"""Run FP8 MLA prefill via mla_prefill_ps_asm_fwd + mla_reduce_v1.
|
||||
|
||||
Q, K, V are already decompressed (post-kv_b_proj), so K and V have
|
||||
``num_heads`` heads (same as Q) and gqa_ratio=1. Writes the
|
||||
result in-place to ``out``, which is the [total_q, nhead * v_head_dim]
|
||||
output buffer supplied by ``forward_mha``; no extra allocation or
|
||||
copy is required.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
total_q = q.shape[0]
|
||||
nhead = self.num_heads
|
||||
v_head_dim = self.v_head_dim
|
||||
tile_q = _FP8_PREFILL_TILE_Q
|
||||
|
||||
# The FP8 ASM kernel expects FP8 inputs; the q_scale/k_scale/v_scale
|
||||
# parameters select per-tensor dequant scales. Q/K/V arrive as
|
||||
# bf16 from kv_b_proj, so cast here (one_scale=1.0 disables scaling).
|
||||
if q.dtype != fp8_dtype:
|
||||
q = q.to(fp8_dtype)
|
||||
if k.dtype != fp8_dtype:
|
||||
k = k.to(fp8_dtype)
|
||||
if v.dtype != fp8_dtype:
|
||||
v = v.to(fp8_dtype)
|
||||
|
||||
one_scale = torch.ones((), dtype=torch.float32, device=q.device)
|
||||
|
||||
# num_partial_tiles is resolved during metadata build to avoid an
|
||||
# in-forward .item() sync that would prevent CUDA Graph capture.
|
||||
# forward_mha gates the FP8 path on fp8_prefill_qo_indptr being set,
|
||||
# and the builder always sets every fp8_prefill_* field together, so
|
||||
# num_partial_tiles is non-None here.
|
||||
num_partial_tiles = attn_metadata.fp8_prefill_num_partial_tiles
|
||||
assert num_partial_tiles is not None
|
||||
|
||||
# Reuse the caller's output buffer to skip the per-call alloc + copy.
|
||||
# The ASM and reduce kernels both write to a [total_q, nhead, v_head_dim]
|
||||
# view, which aliases the [total_q, nhead * v_head_dim] storage of out.
|
||||
out_3d = out.view(total_q, nhead, v_head_dim)
|
||||
|
||||
# Per-call scratch (logits, attn_lse, final_lse) is served from the
|
||||
# workspace manager so allocator churn in the prefill hot path is
|
||||
# bounded after warmup, matching the pattern in PR #41002.
|
||||
logits, attn_lse, final_lse = current_workspace_manager().get_simultaneous(
|
||||
((num_partial_tiles * tile_q, nhead, v_head_dim), torch.float32),
|
||||
((num_partial_tiles * tile_q, nhead), torch.float32),
|
||||
((total_q, nhead), torch.float32),
|
||||
)
|
||||
|
||||
# Phase 1: persistent-scheduling assembly prefill kernel.
|
||||
self._mla_prefill_ps_asm_fwd(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attn_metadata.fp8_prefill_qo_indptr,
|
||||
attn_metadata.fp8_prefill_kv_indptr,
|
||||
attn_metadata.fp8_prefill_kv_indices,
|
||||
attn_metadata.fp8_prefill_work_indptr,
|
||||
attn_metadata.fp8_prefill_work_info_set,
|
||||
attn_metadata.fp8_prefill_max_q_len,
|
||||
self.scale,
|
||||
True, # is_causal
|
||||
logits,
|
||||
attn_lse,
|
||||
out_3d,
|
||||
one_scale,
|
||||
one_scale,
|
||||
one_scale,
|
||||
)
|
||||
|
||||
# Phase 2: reduction across KV splits.
|
||||
self._mla_reduce_v1(
|
||||
logits,
|
||||
attn_lse,
|
||||
attn_metadata.fp8_prefill_reduce_indptr,
|
||||
attn_metadata.fp8_prefill_reduce_final_map,
|
||||
attn_metadata.fp8_prefill_reduce_partial_map,
|
||||
tile_q,
|
||||
out_3d,
|
||||
final_lse,
|
||||
)
|
||||
|
||||
def forward_mha(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: MLACommonMetadata,
|
||||
k_scale: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
"""Dispatch prefill to the FP8 ASM kernel when available.
|
||||
|
||||
Falls back to the parent (``flash_attn_varlen_func``) when FP8
|
||||
MLA prefill is disabled, PS metadata is missing, or chunked
|
||||
context requires two-pass merge.
|
||||
|
||||
The annotation uses the base ``MLACommonMetadata`` to honour LSP
|
||||
with ``MLACommonImpl.forward_mha``; the AITER builder always
|
||||
produces ``AiterMLAMetadata`` instances at runtime, so we narrow
|
||||
with ``isinstance`` before reading the AITER-specific FP8 fields.
|
||||
"""
|
||||
if (
|
||||
not self._fp8_prefill_enabled
|
||||
or not isinstance(attn_metadata, AiterMLAMetadata)
|
||||
or attn_metadata.fp8_prefill_qo_indptr is None
|
||||
):
|
||||
return super().forward_mha(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_c_and_k_pe_cache,
|
||||
attn_metadata,
|
||||
k_scale,
|
||||
output,
|
||||
)
|
||||
|
||||
assert attn_metadata.prefill is not None
|
||||
prefill_metadata = attn_metadata.prefill
|
||||
has_context = prefill_metadata.chunked_context is not None
|
||||
|
||||
if has_context:
|
||||
return super().forward_mha(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
kv_c_and_k_pe_cache,
|
||||
attn_metadata,
|
||||
k_scale,
|
||||
output,
|
||||
)
|
||||
|
||||
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
||||
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
|
||||
)
|
||||
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
k = self._concat_k_nope_k_pe(k_nope, k_pe)
|
||||
|
||||
self._mla_fp8_prefill_attn(q, k, v, attn_metadata, output)
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
|
||||
Reference in New Issue
Block a user