Signed-off-by: Oxana Korzh <okorzh@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2997 lines
122 KiB
Python
2997 lines
122 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import functools
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, ClassVar, Final, cast
|
|
|
|
import torch
|
|
|
|
from vllm import envs
|
|
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
|
from vllm.config import VllmConfig
|
|
from vllm.config.cache import CacheDType
|
|
from vllm.distributed import get_dcp_group
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.layers.attention.mla_attention import (
|
|
MLACommonBackend,
|
|
MLACommonDecodeMetadata,
|
|
MLACommonImpl,
|
|
MLACommonMetadata,
|
|
MLACommonMetadataBuilder,
|
|
QueryLenSupport,
|
|
accumulate_mla_context_chunk,
|
|
init_mla_context_partial,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.triton_utils import tl, triton
|
|
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
|
from vllm.utils.math_utils import cdiv, largest_power_of_2_divisor
|
|
from vllm.v1.attention.backend import (
|
|
AttentionCGSupport,
|
|
AttentionLayer,
|
|
CommonAttentionMetadata,
|
|
MultipleOf,
|
|
max_decode_query_len,
|
|
)
|
|
from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits
|
|
from vllm.v1.attention.backends.utils import (
|
|
get_dcp_local_seq_lens,
|
|
)
|
|
from vllm.v1.attention.ops.rocm_aiter_mla_merge import (
|
|
merge_mla_segments_triton,
|
|
)
|
|
from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
|
|
context_row_indices,
|
|
expand_context,
|
|
gather_compressed_context,
|
|
)
|
|
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
|
|
from vllm.v1.kv_cache_interface import (
|
|
AttentionSpec,
|
|
KVCacheSpec,
|
|
is_quantized_kv_cache,
|
|
)
|
|
from vllm.v1.worker.workspace import (
|
|
current_workspace_manager,
|
|
is_workspace_manager_initialized,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.platforms.interface import DeviceCapability
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def _supports_fused_dcp_prefill_config(
|
|
kv_cache_dtype: str | None,
|
|
query_dtype: torch.dtype,
|
|
kv_lora_rank: int,
|
|
qk_nope_head_dim: int,
|
|
qk_rope_head_dim: int,
|
|
v_head_dim: int,
|
|
) -> bool:
|
|
"""Whether the configuration supports fused DCP context expansion."""
|
|
return (
|
|
kv_cache_dtype in ("fp8", "fp8_e4m3")
|
|
and query_dtype == torch.bfloat16
|
|
and kv_lora_rank == 512
|
|
and qk_nope_head_dim == 128
|
|
and qk_rope_head_dim == 64
|
|
and v_head_dim == 128
|
|
and is_aiter_found_and_supported()
|
|
)
|
|
|
|
|
|
def _segmented_mla_page_size(block_size: int) -> int:
|
|
"""Largest supported power-of-two subpage dividing a physical KV block.
|
|
|
|
The subpage size becomes the segmented kernel's ``TILE_SIZE``: the cache is
|
|
reinterpreted as ``(-1, page_size, 1, head_dim)`` before the call. It must
|
|
divide the physical block and be a power of two, since the kernel walks a
|
|
tile with ``tl.arange(0, TILE_SIZE)``. 128 is the largest tile that tiling
|
|
supports -- a tile holds ``TILE_SIZE x kv_lora_rank`` keys and its scores
|
|
are ``BLOCK_M x TILE_SIZE`` -- so a larger manager block is split into
|
|
several subpages instead of widening the tile.
|
|
"""
|
|
assert block_size > 0
|
|
return min(128, largest_power_of_2_divisor(block_size))
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _get_mla_gluon():
|
|
"""Load the small-head Gluon MLA entry point."""
|
|
unified_module = "aiter.ops.triton.gluon.mla_gluon"
|
|
try:
|
|
from aiter.ops.triton.gluon.mla_gluon import mla_gluon
|
|
|
|
return mla_gluon
|
|
except ModuleNotFoundError as unified_import_error:
|
|
if not unified_module.startswith(unified_import_error.name or ""):
|
|
raise
|
|
legacy_module = "aiter.ops.triton.gluon.mla_decode_gluon"
|
|
try:
|
|
from aiter.ops.triton.gluon.mla_decode_gluon import mla_decode_gluon
|
|
|
|
return mla_decode_gluon
|
|
except ModuleNotFoundError as legacy_import_error:
|
|
if not legacy_module.startswith(legacy_import_error.name or ""):
|
|
raise
|
|
raise RuntimeError(
|
|
"ROCM_AITER_MLA requires an AITER build with the small-head "
|
|
"Gluon MLA kernel (mla_gluon or mla_decode_gluon) when decode "
|
|
"heads are fewer than 16."
|
|
) from unified_import_error
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _fp8_mla_prefill_supported() -> bool:
|
|
"""Auto-detect FP8 MLA prefill via mla_prefill_ps_asm_fwd + mla_reduce_v1.
|
|
|
|
Requires gfx950 plus an AITER build that exports both kernels. When
|
|
either is missing we silently fall back to ``flash_attn_varlen_func``.
|
|
"""
|
|
try:
|
|
from vllm.platforms.rocm import on_gfx950
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
if not on_gfx950():
|
|
return False
|
|
try:
|
|
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1 # noqa: F401
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
return True
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _aiter_mla_native_h24_reducer_supported() -> bool:
|
|
"""Whether AITER's JIT reducer supports the native H24/512 shape."""
|
|
try:
|
|
from aiter.jit.core import AITER_CSRC_DIR
|
|
|
|
reduce_source = Path(AITER_CSRC_DIR) / "kernels" / "mla" / "reduce.cu"
|
|
source = "".join(reduce_source.read_text(encoding="utf-8").split())
|
|
except (ImportError, OSError):
|
|
return False
|
|
return "MLA_REDUCE_CASE_EF(NUM_HEAD,24,HEAD_DIM,512," in source
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _aiter_mla_native_h24_metadata_supported() -> bool:
|
|
"""Whether AITER's fast MLA metadata planner accepts native H24.
|
|
|
|
The reducer and metadata planner have independent shape dispatch. Checking
|
|
only the reducer can route H24 into a planner that rejects it before the
|
|
attention kernel launches. Until AITER exposes a capability API, inspect
|
|
the shipped JIT source for an explicit native-H24 planner branch.
|
|
"""
|
|
try:
|
|
from aiter.jit.core import AITER_CSRC_DIR
|
|
|
|
metadata_source = (
|
|
Path(AITER_CSRC_DIR) / "kernels" / "mla" / "metadata" / "v1_2_device.cuh"
|
|
)
|
|
source = "".join(metadata_source.read_text(encoding="utf-8").split())
|
|
except (ImportError, OSError):
|
|
return False
|
|
return "num_heads==24" in source
|
|
|
|
|
|
def _aiter_mla_native_h24_supported() -> bool:
|
|
"""Whether the complete AITER decode path supports native H24."""
|
|
return (
|
|
_aiter_mla_native_h24_reducer_supported()
|
|
and _aiter_mla_native_h24_metadata_supported()
|
|
)
|
|
|
|
|
|
def _aiter_mla_non_causal_asm_kernels() -> bool:
|
|
"""Whether this arch ships non-causal MLA decode ASM kernels.
|
|
|
|
The Python `causal=` probe cannot see the per-arch manifest. gfx950 has
|
|
the kernels; gfx942 and everything else must fall through to another
|
|
backend.
|
|
"""
|
|
try:
|
|
from vllm.platforms.rocm import on_gfx950
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
return bool(on_gfx950())
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _gluon_mla_decode_supported() -> bool:
|
|
"""The small-head Gluon MLA decode kernel only has a gfx950 (CDNA4) build.
|
|
|
|
Its tiling needs ~160 KiB of LDS, which exceeds CDNA3's 64 KiB, so on
|
|
gfx942 there is no kernel to fall through to and selecting it asserts
|
|
(``mla_gluon requires gfx950``). Restrict Gluon decode to gfx950; other
|
|
archs use the asm persistent decode, which ``get_mla_padded_q`` makes
|
|
correct for any 1..15 heads.
|
|
"""
|
|
try:
|
|
from vllm.platforms.rocm import on_gfx950
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
return on_gfx950()
|
|
|
|
|
|
# Past 2 GiB, mla_gluon swaps its masked buffer_load for an unmasked
|
|
# global_load that reads outside the cache and aborts the process. Mirrors the
|
|
# kernel's own within_2gb bound, so the fallback engages exactly where it does.
|
|
_GLUON_MAX_KV_CACHE_BYTES = 1 << 31
|
|
|
|
|
|
def _gluon_kv_cache_in_bounds(kv_cache_bytes: int | None) -> bool:
|
|
"""Whether this layer's KV cache keeps Gluon on its bounds-checked path.
|
|
|
|
The cache is one flat tensor per layer, so the bound disqualifies the layer
|
|
outright; no batch shape brings an oversized cache back in range. ``None``
|
|
is the not-yet-sized case during profiling, before any kernel runs.
|
|
"""
|
|
if kv_cache_bytes is None or kv_cache_bytes <= _GLUON_MAX_KV_CACHE_BYTES:
|
|
return True
|
|
logger.warning_once(
|
|
"KV cache is %.1f GiB per layer, past the 2 GiB where the Gluon MLA "
|
|
"kernel drops its KV bounds mask; using the padded ASM decode instead. "
|
|
"Lower --gpu-memory-utilization or --max-model-len to get Gluon back.",
|
|
kv_cache_bytes / (1 << 30),
|
|
)
|
|
return False
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _get_segmented_mla_decode():
|
|
"""Load AITER's segmented MLA decode with unreduced partial output."""
|
|
from aiter.ops.triton.attention.mla import mla_decode_fwd
|
|
|
|
return mla_decode_fwd
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _get_aiter_mla_decode():
|
|
from aiter.mla import mla_decode_fwd
|
|
|
|
return mla_decode_fwd
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _segmented_mla_decode_supported() -> bool:
|
|
"""Whether AITER exposes the segmented MLA decode used by DCP verify."""
|
|
try:
|
|
_get_segmented_mla_decode()
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
return True
|
|
|
|
|
|
def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> bool:
|
|
"""Whether this configuration can serve DCP verify on segmented MLA.
|
|
|
|
Configuration only -- whether a given batch takes the route additionally
|
|
depends on its query length. Round-robin interleaving other than 1 is
|
|
excluded because the per-row causal window has not been validated there.
|
|
"""
|
|
return (
|
|
dcp_world_size > 1 and cp_interleave == 1 and _segmented_mla_decode_supported()
|
|
)
|
|
|
|
|
|
def _segmented_mla_fits_lds(num_heads: int, block_size: int, fp8_kv: bool) -> bool:
|
|
"""Whether AITER segmented MLA compiles for this DCP verify shape.
|
|
|
|
On gfx942 it needs 128 KiB of LDS, over the 64 KiB available, for bf16 KV
|
|
at TILE-128 and for fp8 KV past 64 heads, where its query block widens to
|
|
128 heads. ``num_heads`` is the DCP-gathered count.
|
|
"""
|
|
from vllm.platforms.rocm import on_gfx942
|
|
|
|
if not on_gfx942():
|
|
return True
|
|
if fp8_kv:
|
|
return num_heads <= 64
|
|
return _segmented_mla_page_size(block_size) < 128
|
|
|
|
|
|
def _triton_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> bool:
|
|
"""Whether gfx942 can fall back to generic split-KV causal target verify.
|
|
|
|
The batch shape is causal qlen>1 over a DCP shard. Eagle MTP and DSpark
|
|
target verify are that same shape, so this is not limited to one
|
|
speculative method. gfx942 has no cprr kernel, so this serves the shapes
|
|
segmented MLA cannot (see _segmented_mla_fits_lds). qlen==1 and non-causal
|
|
stay on the plain ASM decode.
|
|
"""
|
|
from vllm.platforms.rocm import on_gfx942
|
|
|
|
return on_gfx942() and dcp_world_size > 1 and cp_interleave == 1
|
|
|
|
|
|
# Heads per stage-1 program in the grouped MLA decode kernel (its BLOCK_H).
|
|
_DECODE_HEADS_PER_PROGRAM: Final = 16
|
|
# Tuned on MI325X with microbenchmarks
|
|
_BATCH_OCCUPANCY_MULTIPLIER: Final = 4
|
|
|
|
|
|
def _triton_dcp_verify_num_kv_splits(
|
|
max_seq_len: int, num_rows: int, num_heads: int, sm_count: int
|
|
) -> int:
|
|
# Split only until the stage-1 grid fills the GPU. Past that, splits add
|
|
# scratch and combine work, and large batches overflow the kernel's int32
|
|
# attn_logits offsets.
|
|
programs_per_split = max(1, num_rows) * cdiv(num_heads, _DECODE_HEADS_PER_PROGRAM)
|
|
occupancy_splits = triton.next_power_of_2(
|
|
cdiv(_BATCH_OCCUPANCY_MULTIPLIER * sm_count, programs_per_split)
|
|
)
|
|
return min(_compute_num_kv_splits(max_seq_len, sm_count), occupancy_splits)
|
|
|
|
|
|
def _reserve_triton_dcp_verify_workspace(
|
|
max_rows: int,
|
|
num_heads: int,
|
|
max_seq_len: int,
|
|
kv_lora_rank: int,
|
|
sm_count: int,
|
|
) -> None:
|
|
"""Reserve split-KV scratch before warmup locks the workspace manager."""
|
|
if not is_workspace_manager_initialized():
|
|
return
|
|
# Fewer rows may take more splits, so size for the largest product.
|
|
max_row_splits = max(
|
|
rows * _triton_dcp_verify_num_kv_splits(max_seq_len, rows, num_heads, sm_count)
|
|
for rows in range(1, max_rows + 1)
|
|
)
|
|
current_workspace_manager().get_simultaneous(
|
|
((max_row_splits, num_heads, kv_lora_rank + 1), torch.float32),
|
|
)
|
|
|
|
|
|
# DCP decode routing
|
|
#
|
|
# With decode context parallelism each rank holds a round-robin shard of the KV
|
|
# cache (token t on rank t % dcp_world_size; requires interleave size 1) and
|
|
# attends over it with all DCP-gathered query heads. Partial outputs are merged
|
|
# across ranks by LSE. Each decode batch takes one route:
|
|
#
|
|
# * plain: qlen == 1 or a non-causal block. Every query sees every local token.
|
|
# * cprr: causal qlen >= _MIN_CPRR_QLEN, gfx950, speculative decoding. AITER's
|
|
# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank
|
|
# and cp_world_size and applies the causal window in-kernel. Kernels exist only
|
|
# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up.
|
|
# * triton: causal qlen > 1 on gfx942 for shapes segmented MLA cannot fit in
|
|
# LDS (_segmented_mla_fits_lds). Generic split-KV decode over the same
|
|
# per-row local window segmented uses.
|
|
# * segmented: causal qlen > 1 when cprr is not selected or qlen is below
|
|
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV
|
|
# length encodes its causal window.
|
|
#
|
|
# The plain kernel must never serve a causal qlen > 1 block: it applies
|
|
# causality on local indices, which is wrong over a round-robin shard.
|
|
_NATIVE_CPRR_HEADS: Final = (16, 32, 64, 128)
|
|
_MIN_CPRR_QLEN: Final = 3
|
|
|
|
|
|
class _DCPDecodeRoute(Enum):
|
|
PLAIN = "plain"
|
|
SEGMENTED = "segmented"
|
|
CPRR = "cprr"
|
|
TRITON = "triton"
|
|
|
|
|
|
def _is_row_view_dcp_route(route: _DCPDecodeRoute) -> bool:
|
|
"""Routes that attend one causal window per verify token."""
|
|
return route in (_DCPDecodeRoute.SEGMENTED, _DCPDecodeRoute.TRITON)
|
|
|
|
|
|
def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
|
|
"""Head count the cprr asm decode runs at, or 0 if it cannot serve this shape."""
|
|
if decode_num_heads in _NATIVE_CPRR_HEADS:
|
|
return decode_num_heads
|
|
for nat in _NATIVE_CPRR_HEADS:
|
|
if nat >= decode_num_heads:
|
|
return nat
|
|
return 0
|
|
|
|
|
|
def _select_dcp_decode_route(
|
|
*,
|
|
supports_segmented: bool,
|
|
supports_triton: bool,
|
|
causal: bool,
|
|
max_qo_len: int,
|
|
asm_selected: bool,
|
|
) -> _DCPDecodeRoute:
|
|
"""Select one DCP decode route for this batch; see DCP decode routing."""
|
|
if not causal or max_qo_len <= 1:
|
|
return _DCPDecodeRoute.PLAIN
|
|
if asm_selected and max_qo_len >= _MIN_CPRR_QLEN:
|
|
return _DCPDecodeRoute.CPRR
|
|
if supports_triton:
|
|
return _DCPDecodeRoute.TRITON
|
|
if supports_segmented:
|
|
return _DCPDecodeRoute.SEGMENTED
|
|
raise RuntimeError(
|
|
"ROCM_AITER_MLA DCP multi-token verify requires either segmented "
|
|
"MLA, the round-robin asm decode, or the gfx942 Triton verify path."
|
|
)
|
|
|
|
|
|
def _asm_dcp_verify_configured(
|
|
dcp_world_size: int, cp_interleave: int, multi_token_decode: bool
|
|
) -> bool:
|
|
"""Whether asm cprr DCP verify is reachable in this process.
|
|
|
|
Requires DCP with interleave size 1, speculative decoding, and gfx950 (the
|
|
only arch AITER builds cprr kernels for).
|
|
"""
|
|
if not (dcp_world_size > 1 and cp_interleave == 1 and multi_token_decode):
|
|
return False
|
|
from vllm.platforms.rocm import on_gfx950
|
|
|
|
return bool(on_gfx950())
|
|
|
|
|
|
def _aiter_mla_small_head_mode() -> str:
|
|
"""Small-head (<16) MLA decode kernel selection.
|
|
|
|
Controlled by ``VLLM_ROCM_AITER_MLA_ASM_PADDING``:
|
|
|
|
- ``"auto"`` (default): let the arch decide -- divisor head counts keep the
|
|
Gluon decode where a build exists (gfx950), everything else (non-divisor
|
|
counts and all counts on gfx942) uses the padded persistent-scheduling
|
|
ASM decode.
|
|
- ``"gluon"``: prefer the Gluon path wherever a build exists.
|
|
- ``"asm"``: force the padded persistent-scheduling ASM decode.
|
|
|
|
On gfx942 (no Gluon build) the ASM path is always used regardless of this
|
|
setting; ``"gluon"`` there falls back to ASM with a one-time warning.
|
|
"""
|
|
import vllm.envs as envs
|
|
|
|
mode = (envs.VLLM_ROCM_AITER_MLA_ASM_PADDING or "auto").lower()
|
|
if mode == "gluon" and not _gluon_mla_decode_supported():
|
|
logger.warning_once(
|
|
"VLLM_ROCM_AITER_MLA_ASM_PADDING=gluon requested, but this device "
|
|
"has no Gluon MLA decode build (Gluon requires gfx950); using the "
|
|
"padded persistent-scheduling ASM decode instead."
|
|
)
|
|
return mode
|
|
|
|
|
|
def _adaptive_varlen_verify_enabled(vllm_config: VllmConfig) -> bool:
|
|
speculative_config = vllm_config.speculative_config
|
|
return (
|
|
speculative_config is not None
|
|
and bool(getattr(speculative_config, "enable_adaptive_verification", False))
|
|
and vllm_config.parallel_config.decode_context_parallel_size == 1
|
|
and max_decode_query_len(vllm_config) > 1
|
|
)
|
|
|
|
|
|
class AiterMLABackend(MLACommonBackend):
|
|
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
|
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
|
"auto",
|
|
"float16",
|
|
"bfloat16",
|
|
"fp8",
|
|
"fp8_e4m3",
|
|
"fp8_e5m2",
|
|
]
|
|
|
|
@classmethod
|
|
def get_supported_head_sizes(cls) -> list[int]:
|
|
return []
|
|
|
|
@staticmethod
|
|
def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]:
|
|
# The aiter MLA decode kernel always operates with page_size=1
|
|
# internally (the wrapper flattens kv_buffer via .view(-1, 1, 1, H)).
|
|
# We support any kernel_block_size by expanding block-level indices
|
|
# into per-token flat indices in the metadata builder.
|
|
return [MultipleOf(1)]
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "ROCM_AITER_MLA"
|
|
|
|
@classmethod
|
|
def supports_non_causal(cls) -> bool:
|
|
# causal= on mla_decode_fwd is necessary but not sufficient: pinned
|
|
# AITER v0.1.21.post2 still exports that argument on gfx942, whose ASM
|
|
# manifest has no non-causal decode kernels. Selecting this backend
|
|
# there reaches `cannot get heuristic kernel!` instead of TRITON_MLA.
|
|
return bool(
|
|
rocm_aiter_ops.mla_decode_supports_non_causal()
|
|
and _aiter_mla_non_causal_asm_kernels()
|
|
)
|
|
|
|
@classmethod
|
|
def validate_configuration(
|
|
cls,
|
|
head_size: int,
|
|
dtype: torch.dtype,
|
|
kv_cache_dtype: CacheDType | None,
|
|
block_size: int | None,
|
|
use_mla: bool,
|
|
has_sink: bool,
|
|
use_sparse: bool,
|
|
use_mm_prefix: bool,
|
|
use_per_head_quant_scales: bool,
|
|
device_capability: "DeviceCapability",
|
|
attn_type: str,
|
|
has_sliding_window: bool = False,
|
|
use_non_causal: bool = False,
|
|
use_batch_invariant: bool = False,
|
|
use_kv_connector: bool = False,
|
|
use_pcp: bool = False,
|
|
use_adaptive_verification: bool = False,
|
|
use_dcp: bool = False,
|
|
use_rswa: bool = False,
|
|
) -> list[str]:
|
|
invalid_reasons = super().validate_configuration(
|
|
head_size=head_size,
|
|
dtype=dtype,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
block_size=block_size,
|
|
use_mla=use_mla,
|
|
has_sink=has_sink,
|
|
use_sparse=use_sparse,
|
|
use_mm_prefix=use_mm_prefix,
|
|
use_per_head_quant_scales=use_per_head_quant_scales,
|
|
device_capability=device_capability,
|
|
attn_type=attn_type,
|
|
has_sliding_window=has_sliding_window,
|
|
use_non_causal=use_non_causal,
|
|
use_batch_invariant=use_batch_invariant,
|
|
use_kv_connector=use_kv_connector,
|
|
use_pcp=use_pcp,
|
|
use_adaptive_verification=use_adaptive_verification,
|
|
use_dcp=use_dcp,
|
|
use_rswa=use_rswa,
|
|
)
|
|
# Arch/signature cannot express this: the backend still advertises
|
|
# fp16, so a DSpark draft with kv_cache_dtype="auto" would select
|
|
# AITER and abort on fp16 query. Causal fp16 is fine.
|
|
if (
|
|
use_non_causal
|
|
and dtype == torch.float16
|
|
and kv_cache_dtype in (None, "auto", "float16")
|
|
):
|
|
invalid_reasons.append("non-causal fp16 MLA decode not supported")
|
|
return invalid_reasons
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type["AiterMLAImpl"]:
|
|
return AiterMLAImpl
|
|
|
|
@staticmethod
|
|
def get_builder_cls() -> type["AiterMLAMetadataBuilder"]:
|
|
return AiterMLAMetadataBuilder
|
|
|
|
|
|
@dataclass
|
|
class AiterMLADCPVerifyMetadata:
|
|
"""One paged-KV row per verify token for causal DCP verification.
|
|
|
|
Segmented MLA and the gfx942 Triton path both consume this view. Which
|
|
kernel runs is ``AiterMLADecodeMetadata.dcp_route``.
|
|
"""
|
|
|
|
# Local KV length of each verify row, already causally bounded.
|
|
row_lens: torch.Tensor
|
|
# Subpage IDs each row reads, shape [num_rows, max_local_pages].
|
|
block_table: torch.Tensor
|
|
# One query per row, so this is an arange.
|
|
qo_indptr: torch.Tensor
|
|
# Subpage size the block table is expressed in, and the segmented kernel's
|
|
# TILE_SIZE. Carried so the impl reinterprets the cache exactly the way the
|
|
# block table was built, instead of re-deriving it from the cache shape.
|
|
page_size: int
|
|
# Kernel-visible KV bound. Whenever full graphs are enabled this is the
|
|
# configuration's maximum for every batch, not just during capture, so the
|
|
# page table keeps one shape across replays.
|
|
max_kv_seq_len: int
|
|
|
|
|
|
@dataclass
|
|
class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
|
|
# The indptr of the paged kv cache, shape: [batch_size + 1]
|
|
paged_kv_indptr: torch.Tensor | None = None
|
|
# The page indices of the paged kv cache
|
|
paged_kv_indices: torch.Tensor | None = None
|
|
# The number of entries in the last page of each request in
|
|
# the paged kv cache, shape: [batch_size]
|
|
paged_kv_last_page_len: torch.Tensor | None = None
|
|
# The query indptr, shape : [num_decode + 1]
|
|
qo_indptr: torch.Tensor | None = None
|
|
# The dtype of MLA out tensor
|
|
attn_out_dtype: torch.dtype = torch.bfloat16
|
|
# The max query output length: int
|
|
max_qo_len: int | None = None
|
|
# Minimum KV length used by Gluon to choose a safe split count.
|
|
min_kv_seq_len: int = 1
|
|
# Exactly one DCP decode route is selected by the metadata builder.
|
|
dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN
|
|
# Set when this batch takes a per-row DCP verify kernel (segmented MLA or
|
|
# gfx942 Triton). ``dcp_route`` selects which kernel.
|
|
dcp_verify: AiterMLADCPVerifyMetadata | None = None
|
|
# cprr only: global per-request page indptr and this rank's position.
|
|
g_kv_indptr: torch.Tensor | None = None
|
|
cp_world_size: int = 1
|
|
cp_rank: int = 0
|
|
# Padded head count cprr runs at; 0 when this batch is not on cprr.
|
|
asm_decode_num_heads: int = 0
|
|
# Must equal the split cap the reduce scratch was sized with, or the kernel
|
|
# writes past it and faults.
|
|
mla_num_kv_splits: int = 0
|
|
# Small-head decode uses Gluon (avoids padding to 16).
|
|
use_gluon_decode: bool = False
|
|
# Small-head non-DCP multi-token verify uses Gluon's native MTP entry.
|
|
use_gluon_verify: bool = False
|
|
# Whether persistent MLA metadata was computed
|
|
has_persistent_metadata: bool = False
|
|
varlen_q_rows: torch.Tensor | None = None
|
|
varlen_o_rows: torch.Tensor | None = None
|
|
|
|
|
|
@dataclass
|
|
class AiterMLAMetadata(MLACommonMetadata[AiterMLADecodeMetadata]):
|
|
dcp_context_row_indices: list[torch.Tensor] | None = None
|
|
work_meta_data: torch.Tensor | None = None
|
|
work_indptr: torch.Tensor | None = None
|
|
work_info_set: torch.Tensor | None = None
|
|
reduce_indptr: torch.Tensor | None = None
|
|
reduce_final_map: torch.Tensor | None = None
|
|
reduce_partial_map: torch.Tensor | None = None
|
|
|
|
# FP8 ASM prefill persistent-scheduling (PS) metadata. Populated by
|
|
# AiterMLAMetadataBuilder._build_fp8_prefill_ps_metadata when prefill
|
|
# tokens are present and FP8 MLA prefill is supported on the device.
|
|
# Left as None on hosts/configs that fall back to flash_attn_varlen_func.
|
|
fp8_prefill_qo_indptr: torch.Tensor | None = None
|
|
fp8_prefill_kv_indptr: torch.Tensor | None = None
|
|
fp8_prefill_kv_indices: torch.Tensor | None = None
|
|
fp8_prefill_work_indptr: torch.Tensor | None = None
|
|
fp8_prefill_work_info_set: torch.Tensor | None = None
|
|
fp8_prefill_reduce_indptr: torch.Tensor | None = None
|
|
fp8_prefill_reduce_final_map: torch.Tensor | None = None
|
|
fp8_prefill_reduce_partial_map: torch.Tensor | None = None
|
|
fp8_prefill_max_q_len: int | None = None
|
|
fp8_prefill_num_partial_tiles: int | None = None
|
|
|
|
|
|
# Tile size used by the mla_prefill_ps_asm_fwd assembly kernel.
|
|
_FP8_PREFILL_TILE_Q = 256
|
|
|
|
|
|
class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
|
# TODO(luka, lucas): audit this as part of:
|
|
# https://github.com/vllm-project/vllm/issues/22945
|
|
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
|
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
|
|
# Served by passing the mask to the kernel; _build_decode turns away the
|
|
# shapes AITER has no non-causal kernel for.
|
|
supports_non_causal_multi_token_decode: ClassVar[bool] = True
|
|
# Set from the common metadata every build; a batch is causal unless the
|
|
# drafter says otherwise.
|
|
_decode_causal: bool = True
|
|
# A non-causal draft needs no cross-shard causal window: the ordinary
|
|
# mask0 decode returns per-row LSE for the DCP merge.
|
|
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
|
|
|
|
@staticmethod
|
|
def _uniform_padded_mtp_qo_len(
|
|
qo_len: torch.Tensor,
|
|
max_qo_len: int,
|
|
num_decode_tokens: int,
|
|
) -> int:
|
|
num_reqs = qo_len.numel()
|
|
if num_reqs == 0 or num_decode_tokens <= 0:
|
|
return 0
|
|
|
|
# Full-CG pads q to a captured token count while leaving
|
|
# query_start_loc flat for dummy requests. Only synthesize dummy rows
|
|
# when every padded request maps to the same qlen and the q buffer has
|
|
# exactly that many rows.
|
|
if num_decode_tokens % num_reqs != 0:
|
|
return 0
|
|
|
|
uniform_qo_len = num_decode_tokens // num_reqs
|
|
if uniform_qo_len <= 1:
|
|
return 0
|
|
|
|
# qo_len is derived from query_start_loc_cpu, so it is already
|
|
# host-side and tiny (one entry per request). Reading it out once and
|
|
# deciding on Python ints avoids eight separate tensor dispatches --
|
|
# sum/item, boolean-mask index, all, eq, nonzero, item, any -- on the
|
|
# metadata-build path, which runs between decode steps while the GPUs
|
|
# are idle.
|
|
lens = qo_len.tolist()
|
|
if num_decode_tokens <= sum(lens):
|
|
return 0
|
|
|
|
first_zero = -1
|
|
num_positive = 0
|
|
for i, qlen in enumerate(lens):
|
|
if qlen > 0:
|
|
num_positive += 1
|
|
# A positive entry after a zero breaks the contiguous live
|
|
# prefix that the padded layout requires.
|
|
if first_zero >= 0:
|
|
return 0
|
|
elif first_zero < 0:
|
|
first_zero = i
|
|
|
|
if num_positive == num_reqs:
|
|
return 0
|
|
if num_positive > 0:
|
|
if max_qo_len != uniform_qo_len:
|
|
return 0
|
|
if any(qlen != uniform_qo_len for qlen in lens if qlen > 0):
|
|
return 0
|
|
|
|
return uniform_qo_len
|
|
|
|
@classmethod
|
|
def get_varlen_cudagraph_max_query_len(
|
|
cls: type["AiterMLAMetadataBuilder"],
|
|
vllm_config: VllmConfig,
|
|
kv_cache_spec: KVCacheSpec,
|
|
) -> int | None:
|
|
if not _adaptive_varlen_verify_enabled(vllm_config):
|
|
return None
|
|
if (
|
|
cls.get_cudagraph_support(vllm_config, kv_cache_spec)
|
|
!= AttentionCGSupport.UNIFORM_BATCH
|
|
):
|
|
return None
|
|
return max_decode_query_len(vllm_config)
|
|
|
|
def __init__(
|
|
self,
|
|
kv_cache_spec: AttentionSpec,
|
|
layer_names: list[str],
|
|
vllm_config: VllmConfig,
|
|
device: torch.device,
|
|
):
|
|
parallel_config = vllm_config.parallel_config
|
|
supports_segmented_dcp_verify = _segmented_dcp_verify_supported(
|
|
parallel_config.decode_context_parallel_size,
|
|
parallel_config.cp_kv_cache_interleave_size,
|
|
)
|
|
supports_triton_dcp_verify = _triton_dcp_verify_supported(
|
|
parallel_config.decode_context_parallel_size,
|
|
parallel_config.cp_kv_cache_interleave_size,
|
|
)
|
|
verify_route = envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY.lower()
|
|
asm_dcp_verify_config = (
|
|
_asm_dcp_verify_configured(
|
|
parallel_config.decode_context_parallel_size,
|
|
parallel_config.cp_kv_cache_interleave_size,
|
|
multi_token_decode=vllm_config.speculative_config is not None,
|
|
)
|
|
and verify_route != "segmented"
|
|
)
|
|
super().__init__(
|
|
kv_cache_spec,
|
|
layer_names,
|
|
vllm_config,
|
|
device,
|
|
AiterMLAMetadata,
|
|
supports_dcp_with_varlen=(
|
|
supports_segmented_dcp_verify
|
|
or asm_dcp_verify_config
|
|
or supports_triton_dcp_verify
|
|
),
|
|
)
|
|
self._asm_dcp_verify = False
|
|
self._asm_dcp_verify_heads = 0
|
|
# The head-count half of the route decision needs self.num_heads, so only
|
|
# the config half is known before super().__init__().
|
|
if asm_dcp_verify_config:
|
|
self._asm_dcp_verify_heads = _asm_dcp_verify_heads(
|
|
self.num_heads * self.dcp_world_size
|
|
)
|
|
if self._asm_dcp_verify_heads:
|
|
self._asm_dcp_verify = True
|
|
elif verify_route == "asm" or not supports_segmented_dcp_verify:
|
|
raise ValueError(
|
|
f"VLLM_ROCM_AITER_MLA_DCP_VERIFY={verify_route}, but the "
|
|
"round-robin asm decode has no kernel for "
|
|
f"{self.num_heads * self.dcp_world_size} DCP-gathered heads "
|
|
f"(native counts: {_NATIVE_CPRR_HEADS}). "
|
|
+ (
|
|
"Set VLLM_ROCM_AITER_MLA_DCP_VERIFY=segmented."
|
|
if supports_segmented_dcp_verify
|
|
else "This AITER build also lacks segmented MLA decode."
|
|
)
|
|
)
|
|
self._supports_segmented_dcp_verify = (
|
|
supports_segmented_dcp_verify
|
|
and _segmented_mla_fits_lds(
|
|
self.num_heads * self.dcp_world_size,
|
|
kv_cache_spec.block_size,
|
|
is_quantized_kv_cache(vllm_config.cache_config.cache_dtype),
|
|
)
|
|
)
|
|
self._use_triton_dcp_verify = (
|
|
supports_triton_dcp_verify and not self._supports_segmented_dcp_verify
|
|
)
|
|
self._mla_max_split_per_batch = 0
|
|
if self._asm_dcp_verify:
|
|
# Cap on KV splits per batch; at low batch the KV axis is the only
|
|
# parallelism. Buffer sizing and every runtime call must see this value.
|
|
self._mla_max_split_per_batch = torch.cuda.get_device_properties(
|
|
device
|
|
).multi_processor_count
|
|
|
|
# Use the group's original format: decode normalizes E5M2 to "fp8".
|
|
kv_cache_dtype = getattr(kv_cache_spec, "cache_dtype_str", None)
|
|
if kv_cache_dtype is None:
|
|
kv_cache_dtype = vllm_config.cache_config.cache_dtype
|
|
self._supports_fused_dcp_prefill = _supports_fused_dcp_prefill_config(
|
|
kv_cache_dtype,
|
|
self.q_data_type,
|
|
self.mla_dims.kv_lora_rank,
|
|
self.mla_dims.qk_nope_head_dim,
|
|
self.mla_dims.qk_rope_head_dim,
|
|
self.mla_dims.v_head_dim,
|
|
)
|
|
|
|
self.compilation_config = vllm_config.compilation_config
|
|
self.decode_attn_out_dtype = vllm_config.model_config.dtype
|
|
|
|
# Needed to place a verify row's causal window on this rank's KV shard.
|
|
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
|
|
|
# reorder_batch_threshold is the largest query length decode can be
|
|
# handed, and already accounts for the drafting scheme. A method-name
|
|
# whitelist sizes unlisted drafters for qlen=1, which closes the
|
|
# persistent gate below and makes aiter raise a KeyError mid-run.
|
|
self._mtp_decode_qlen = self.reorder_batch_threshold or 1
|
|
|
|
self._varlen_decode_qlen = 0
|
|
if _adaptive_varlen_verify_enabled(vllm_config):
|
|
assert self._mtp_decode_qlen == max_decode_query_len(vllm_config), (
|
|
self._mtp_decode_qlen,
|
|
max_decode_query_len(vllm_config),
|
|
)
|
|
self._varlen_decode_qlen = self._mtp_decode_qlen
|
|
self.query_len_support = QueryLenSupport.VARLEN # type: ignore[misc]
|
|
max_rows = vllm_config.scheduler_config.max_num_seqs * (
|
|
self._varlen_decode_qlen
|
|
)
|
|
row_ids = torch.arange(max_rows, dtype=torch.int64, device=device)
|
|
self._varlen_row_req = row_ids // self._varlen_decode_qlen
|
|
self._varlen_row_pos = row_ids % self._varlen_decode_qlen
|
|
self._varlen_row_ids = row_ids
|
|
self._varlen_q_rows = torch.zeros_like(row_ids)
|
|
self._varlen_o_rows = torch.zeros(
|
|
max_rows + 1, dtype=torch.int64, device=device
|
|
)
|
|
|
|
# Store the kernel block size from the spec. When kernel_block_size=1
|
|
# (no spec-dec), behavior is identical to the original. When > 1
|
|
# (e.g. 16 with Eagle3), we expand block-level indices into per-token
|
|
# flat indices since the aiter kernel always uses page_size=1 internally.
|
|
self.kernel_block_size = kv_cache_spec.block_size
|
|
# Generic Triton indexes physical KV blocks; segmented MLA indexes
|
|
# TILE subpages of them.
|
|
self._dcp_verify_page_size = (
|
|
kv_cache_spec.block_size
|
|
if self._use_triton_dcp_verify
|
|
else _segmented_mla_page_size(kv_cache_spec.block_size)
|
|
)
|
|
# A DCP rank's shard of the longest sequence bounds every verify row.
|
|
dcp_local_max_seq_len = (
|
|
cdiv(
|
|
vllm_config.model_config.max_model_len,
|
|
self.dcp_world_size * parallel_config.cp_kv_cache_interleave_size,
|
|
)
|
|
* parallel_config.cp_kv_cache_interleave_size
|
|
)
|
|
|
|
# In the flat view (.view(-1,1,1,H)), each token is its own page,
|
|
# so max_num_pages_per_req = max_model_len regardless of
|
|
# kernel_block_size.
|
|
max_num_pages_per_req = vllm_config.model_config.max_model_len
|
|
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
|
max_num_pages = max_num_reqs * max_num_pages_per_req
|
|
speculative_config = vllm_config.speculative_config
|
|
if _adaptive_varlen_verify_enabled(vllm_config):
|
|
assert speculative_config is not None
|
|
max_num_pages += max_num_reqs * (
|
|
speculative_config.num_speculative_tokens or 0
|
|
)
|
|
|
|
# Preparing persistent buffers
|
|
# TODO: we can disambiguate between decode and mixed-prefill decode here
|
|
# so we can only use the persistent buffer if a cudagraph is actually
|
|
# being used.
|
|
|
|
# paged_kv_last_page_len is always 1s (the aiter kernel always sees
|
|
# page_size=1 after .view(-1,1,1,H) flattening), so we create it
|
|
# once and reuse slices in both eager and cudagraph modes.
|
|
self.paged_kv_last_page_len = torch.ones(
|
|
max_num_reqs, dtype=torch.int32, device=device
|
|
)
|
|
|
|
# Persistent buffer for paged_kv_indices to avoid blocking boolean mask
|
|
# indexing (block_table_tensor[mask]) which has data-dependent output size.
|
|
self.paged_kv_indices = torch.zeros(
|
|
max_num_pages, dtype=torch.int32, device=device
|
|
)
|
|
|
|
from aiter import dtypes, get_mla_metadata_info_v1
|
|
|
|
# Decode kernels consume the DCP-gathered query heads.
|
|
self._decode_num_heads = self.num_heads * self.dcp_world_size
|
|
# Keep metadata sizing consistent with the padded tensor shape passed
|
|
# to mla_decode_fwd, including native 24-head AITER builds.
|
|
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
|
|
self._decode_num_heads
|
|
)
|
|
# the cprr kernel exists only at the native head
|
|
# counts, so run (and size the persistent schedule) at the padded count.
|
|
# 96 is 16-aligned, so get_actual_mla_num_heads leaves it alone and the
|
|
# pad has to be applied here explicitly.
|
|
if self._asm_dcp_verify:
|
|
self._num_attention_heads = self._asm_dcp_verify_heads
|
|
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
|
|
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
|
|
kv_cache_dtype_str = "fp8"
|
|
kv_dtype = dtypes.fp8
|
|
else:
|
|
kv_dtype = {
|
|
torch.float16: dtypes.fp16,
|
|
torch.bfloat16: dtypes.bf16,
|
|
}[kv_cache_spec.dtype]
|
|
# _build_decode needs the cache dtype to pick the decode kernel; keep
|
|
# the normalized string instead of dropping it at the end of __init__.
|
|
self._kv_cache_dtype_str = kv_cache_dtype_str
|
|
# Sized per layer, matching the flat KV tensor the kernel is handed.
|
|
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks
|
|
self._kv_cache_bytes = (
|
|
None
|
|
if num_gpu_blocks is None
|
|
else num_gpu_blocks * kv_cache_spec.page_size_bytes
|
|
)
|
|
# MLAAttention quantizes decode Q to FP8 before calling this backend
|
|
# whenever the KV cache is FP8 and supports_quant_query_input is true.
|
|
q_dtype = (
|
|
dtypes.fp8 if kv_cache_dtype_str == "fp8" else self.decode_attn_out_dtype
|
|
)
|
|
# Persist for get_mla_metadata_v1 (decode build): omitting these causes
|
|
# wrong split/reduce metadata for the gfx950 fp8 nhead=32 fold path.
|
|
self._mla_q_dtype = q_dtype
|
|
self._mla_kv_dtype = kv_dtype
|
|
(
|
|
(work_meta_data_size, work_meta_data_type),
|
|
(work_indptr_size, work_indptr_type),
|
|
(work_info_set_size, work_info_set_type),
|
|
(reduce_indptr_size, reduce_indptr_type),
|
|
(reduce_final_map_size, reduce_final_map_type),
|
|
(reduce_partial_map_size, reduce_partial_map_type),
|
|
) = get_mla_metadata_info_v1(
|
|
max_num_reqs,
|
|
self._mtp_decode_qlen,
|
|
self._num_attention_heads,
|
|
q_dtype,
|
|
kv_dtype,
|
|
is_sparse=False,
|
|
fast_mode=True,
|
|
# Must match max_split_per_batch in get_mla_metadata_v1; without it
|
|
# aiter sizes the reduce scratch from capacity (~9 GiB).
|
|
**(
|
|
dict(max_split_per_batch=self._mla_max_split_per_batch)
|
|
if self._asm_dcp_verify
|
|
else {}
|
|
),
|
|
)
|
|
self._mla_work_meta_data = torch.empty(
|
|
work_meta_data_size, dtype=work_meta_data_type, device=device
|
|
)
|
|
self._mla_work_indptr = torch.empty(
|
|
work_indptr_size, dtype=work_indptr_type, device=device
|
|
)
|
|
self._mla_work_info_set = torch.empty(
|
|
work_info_set_size, dtype=work_info_set_type, device=device
|
|
)
|
|
self._mla_reduce_indptr = torch.empty(
|
|
reduce_indptr_size, dtype=reduce_indptr_type, device=device
|
|
)
|
|
self._mla_reduce_final_map = torch.empty(
|
|
reduce_final_map_size, dtype=reduce_final_map_type, device=device
|
|
)
|
|
self._mla_reduce_partial_map = torch.empty(
|
|
reduce_partial_map_size,
|
|
dtype=reduce_partial_map_type,
|
|
device=device,
|
|
)
|
|
|
|
# The assembly prefill requires FP8 KV, bf16 output, and 16-aligned
|
|
# heads. It writes bf16 through a raw output pointer, so fp16 must use
|
|
# the standard prefill path. Head counts that are not a multiple of 16
|
|
# are replicate-padded up to one in _mla_fp8_prefill_attn, so the gate
|
|
# is the same head-count predicate the decode path uses.
|
|
self._fp8_prefill_enabled = _fp8_mla_prefill_supported() and (
|
|
kv_cache_dtype_str == "fp8"
|
|
and vllm_config.model_config.dtype == torch.bfloat16
|
|
and AiterMLAHelper.is_valid_num_heads(self.num_heads)
|
|
)
|
|
if self._fp8_prefill_enabled:
|
|
max_prefill_qlen = min(
|
|
vllm_config.model_config.max_model_len,
|
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
)
|
|
self._init_fp8_prefill_ps_buffers(
|
|
max_num_reqs,
|
|
max_prefill_qlen,
|
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
vllm_config.model_config.dtype,
|
|
device,
|
|
)
|
|
|
|
# Full cudagraphs need a stable address for the cprr global page indptr.
|
|
# Zero-initialized; element 0 is the indptr base and is never rewritten.
|
|
self._g_kv_indptr_buf: torch.Tensor | None = None
|
|
if self._asm_dcp_verify:
|
|
self._g_kv_indptr_buf = torch.zeros(
|
|
max_num_reqs + 1, dtype=torch.int32, device=device
|
|
)
|
|
|
|
# Persistent buffers for segmented DCP verification. Captured graphs
|
|
# require stable addresses while row lengths vary between replays.
|
|
self._dcp_verify_buffers: AiterMLADCPVerifyMetadata | None = None
|
|
self._graph_seq_lens: torch.Tensor | None = None
|
|
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
|
|
self.paged_kv_indptr = torch.zeros(
|
|
max_num_reqs + 1, dtype=torch.int32, device=device
|
|
)
|
|
|
|
self.qo_indptr = torch.zeros(
|
|
max_num_reqs + 1, dtype=torch.int32, device=device
|
|
)
|
|
|
|
# Full graphs require a stable address after uniform-MTP padding.
|
|
self._graph_seq_lens = torch.zeros(
|
|
max_num_reqs, dtype=torch.int32, device=device
|
|
)
|
|
|
|
if (
|
|
self._supports_segmented_dcp_verify or self._use_triton_dcp_verify
|
|
) and self._mtp_decode_qlen > 1:
|
|
# Allocate even when CPRR is the preferred route: a later
|
|
# replay can fall below _MIN_CPRR_QLEN and needs these
|
|
# addresses. Full graphs need the row bound to be constant.
|
|
max_verify_rows = max_num_reqs * self._mtp_decode_qlen
|
|
max_local_pages = cdiv(
|
|
dcp_local_max_seq_len, self._dcp_verify_page_size
|
|
)
|
|
self._dcp_verify_buffers = AiterMLADCPVerifyMetadata(
|
|
row_lens=torch.zeros(
|
|
max_verify_rows, dtype=torch.int32, device=device
|
|
),
|
|
block_table=torch.zeros(
|
|
(max_verify_rows, max_local_pages),
|
|
dtype=torch.int32,
|
|
device=device,
|
|
),
|
|
qo_indptr=torch.arange(
|
|
max_verify_rows + 1, dtype=torch.int32, device=device
|
|
),
|
|
page_size=self._dcp_verify_page_size,
|
|
max_kv_seq_len=dcp_local_max_seq_len,
|
|
)
|
|
# gfx942 DCP verify launches the generic split-KV kernel, which draws
|
|
# its partials from the shared workspace. Reserve it before warmup
|
|
# locks the pool.
|
|
if self._use_triton_dcp_verify and self._mtp_decode_qlen > 1:
|
|
_reserve_triton_dcp_verify_workspace(
|
|
max_num_reqs * self._mtp_decode_qlen,
|
|
self._decode_num_heads,
|
|
dcp_local_max_seq_len,
|
|
self.mla_dims.kv_lora_rank,
|
|
current_platform.num_compute_units(),
|
|
)
|
|
|
|
def _init_fp8_prefill_ps_buffers(
|
|
self,
|
|
max_num_reqs: int,
|
|
max_prefill_qlen: int,
|
|
max_num_batched_tokens: int,
|
|
attn_out_dtype: torch.dtype,
|
|
device: torch.device,
|
|
) -> None:
|
|
"""Pre-allocate persistent buffers for FP8 MLA prefill PS metadata.
|
|
|
|
Uses ``get_ps_metadata_info_v1`` with max values so the buffers are
|
|
large enough for any batch. ``get_ps_metadata_v1`` fills them
|
|
per-batch in ``build()``. The FP8 prefill forward path also uses the
|
|
global workspace manager for per-call scratch, so reserve its maximum
|
|
shape here before the workspace manager is locked after warmup.
|
|
|
|
Args:
|
|
max_num_reqs: Maximum number of concurrent requests.
|
|
max_prefill_qlen: Maximum Q-length for a single request in one
|
|
prefill batch. Should be ``min(max_model_len,
|
|
max_num_batched_tokens)`` — a single request never exceeds
|
|
``max_model_len`` tokens, nor the per-batch token budget.
|
|
max_num_batched_tokens: Maximum number of tokens scheduled in one
|
|
batch. The ``final_lse`` scratch is sized by ``total_q`` (the
|
|
summed Q-length over all prefill requests in the batch), which
|
|
is bounded by this budget rather than by a single request's
|
|
``max_prefill_qlen`` — concurrent requests can sum to more than
|
|
``max_model_len`` when ``max_model_len < max_num_batched_tokens``.
|
|
attn_out_dtype: Dtype of the attention output buffer, used to size
|
|
the padded-head output scratch (small head counts only).
|
|
device: Target device for the buffers.
|
|
|
|
"""
|
|
from aiter import get_ps_metadata_info_v1
|
|
|
|
# After kv_b_proj decompression, K has num_heads heads (same as Q).
|
|
# So gqa_ratio=1 and num_head_k=num_heads for the PS kernel.
|
|
# Head counts that are not a multiple of 16 (K3: 12/rank at TP8) are
|
|
# replicate-padded up to one in _mla_fp8_prefill_attn; build the PS
|
|
# metadata for that same padded count so the work/reduce maps and the
|
|
# scratch reservations describe the width the kernel is handed.
|
|
#
|
|
# This was previously max(16, num_heads), which agrees with the helper
|
|
# at every head count reachable today (12 and 16 both give 16) and
|
|
# differs only above 16: at 24 it leaves num_head_k=24 while the
|
|
# forward pads to 32. That is not a correctness bug -- the 24-wide work
|
|
# maps still cover head-tiles 0..23, which are the real heads -- but it
|
|
# is expensive, because a lower head alignment yields more partial
|
|
# tiles: gcd-driven, 24 heads -> 64 tiles vs 32 heads -> 16, i.e.
|
|
# 193.6 MiB of reservations instead of 68.6 MiB at batch=1/qlen=512
|
|
# (measured on gfx950). Sizing both from one helper keeps the widths
|
|
# equal and takes the cheaper tiling.
|
|
num_head_k = AiterMLAHelper.get_fp8_prefill_num_heads(self.num_heads)
|
|
v_head_dim = self.mla_dims.v_head_dim
|
|
# gqa_ratio = 1
|
|
# qlen_granularity = _FP8_PREFILL_TILE_Q // max(gqa_ratio, 1)
|
|
qlen_granularity = _FP8_PREFILL_TILE_Q
|
|
|
|
(
|
|
(work_metadata_size, work_metadata_dtype),
|
|
(work_indptr_size, work_indptr_dtype),
|
|
(work_info_size, work_info_dtype),
|
|
(reduce_indptr_size, reduce_indptr_dtype),
|
|
(reduce_final_map_size, reduce_final_map_dtype),
|
|
(reduce_partial_map_size, reduce_partial_map_dtype),
|
|
) = get_ps_metadata_info_v1(
|
|
batch_size=max_num_reqs,
|
|
num_head_k=num_head_k,
|
|
max_qlen=max_prefill_qlen,
|
|
qlen_granularity=qlen_granularity,
|
|
)
|
|
|
|
self.fp8_ps_work_metadata = torch.empty(
|
|
work_metadata_size, dtype=work_metadata_dtype, device=device
|
|
)
|
|
self.fp8_ps_work_indptr = torch.empty(
|
|
work_indptr_size, dtype=work_indptr_dtype, device=device
|
|
)
|
|
self.fp8_ps_work_info = torch.empty(
|
|
*work_info_size, dtype=work_info_dtype, device=device
|
|
)
|
|
self.fp8_ps_reduce_indptr = torch.empty(
|
|
reduce_indptr_size, dtype=reduce_indptr_dtype, device=device
|
|
)
|
|
self.fp8_ps_reduce_final_map = torch.empty(
|
|
*reduce_final_map_size, dtype=reduce_final_map_dtype, device=device
|
|
)
|
|
self.fp8_ps_reduce_partial_map = torch.empty(
|
|
reduce_partial_map_size,
|
|
dtype=reduce_partial_map_dtype,
|
|
device=device,
|
|
)
|
|
|
|
# get_ps_metadata_v1 builds the plan on the host and writes its outputs with
|
|
# blocking hipMemcpy that is not ordered on the current stream. Writing the
|
|
# device buffers directly lets step N+1's build (async scheduling) overwrite a
|
|
# plan that step N's prefill kernels are still reading. Build into pinned host
|
|
# staging instead and copy on the current stream. Two staging slots, each
|
|
# reused only after its previous H2D copy has completed.
|
|
self._fp8_ps_device_outputs = (
|
|
self.fp8_ps_work_indptr,
|
|
self.fp8_ps_work_info,
|
|
self.fp8_ps_reduce_indptr,
|
|
self.fp8_ps_reduce_final_map,
|
|
self.fp8_ps_reduce_partial_map,
|
|
)
|
|
self._fp8_ps_staging = [
|
|
tuple(
|
|
torch.empty(t.shape, dtype=t.dtype, device="cpu", pin_memory=True)
|
|
for t in self._fp8_ps_device_outputs
|
|
)
|
|
for _ in range(2)
|
|
]
|
|
# One event per slot, recorded after that slot's H2D copy. synchronize()
|
|
# on an event that has not been recorded yet returns immediately.
|
|
self._fp8_ps_staging_free = [torch.cuda.Event(), torch.cuda.Event()]
|
|
self._fp8_ps_slot = 0
|
|
|
|
from vllm.platforms import current_platform
|
|
from vllm.v1.worker.workspace import current_workspace_manager
|
|
|
|
# AITER metadata sizing assumes all requests can carry max_prefill_qlen tokens,
|
|
# which is a loose worst case in the number of QO tiles. Rather, the sum of
|
|
# qlens is bounded by max_num_batched_tokens. Manually compute the number of
|
|
# qo tiles to avoid OOM at startup.
|
|
# TODO: AITER should give us this budget constrained value
|
|
qo_tile_cnt = (
|
|
cdiv(max_num_batched_tokens, _FP8_PREFILL_TILE_Q) + max_num_reqs - 1
|
|
)
|
|
max_num_partial_tiles = qo_tile_cnt + current_platform.num_compute_units()
|
|
reservations: list[tuple[tuple[int, ...], torch.dtype]] = [
|
|
(
|
|
(max_num_partial_tiles * _FP8_PREFILL_TILE_Q, num_head_k, v_head_dim),
|
|
torch.float32,
|
|
),
|
|
(
|
|
(max_num_partial_tiles * _FP8_PREFILL_TILE_Q, num_head_k),
|
|
torch.float32,
|
|
),
|
|
((max_num_batched_tokens, num_head_k), torch.float32),
|
|
]
|
|
if self.num_heads < num_head_k:
|
|
# Padded head counts also take their kernel output buffer from the
|
|
# workspace: the caller's [total_q, num_heads * v_head_dim] output
|
|
# cannot back a num_head_k-head view (see _mla_fp8_prefill_attn).
|
|
reservations.append(
|
|
((max_num_batched_tokens, num_head_k, v_head_dim), attn_out_dtype)
|
|
)
|
|
current_workspace_manager().get_simultaneous(*reservations)
|
|
|
|
logger.info(
|
|
"FP8 MLA prefill PS buffers allocated "
|
|
"(max_batch=%d, max_qlen=%d, num_head_k=%d)",
|
|
max_num_reqs,
|
|
max_prefill_qlen,
|
|
num_head_k,
|
|
)
|
|
|
|
def _build_fp8_prefill_ps_metadata(
|
|
self,
|
|
metadata: AiterMLAMetadata,
|
|
common_attn_metadata: CommonAttentionMetadata,
|
|
) -> None:
|
|
"""Build per-batch FP8 MLA prefill PS metadata and attach to *metadata*.
|
|
|
|
Called from ``build()`` when prefill tokens are present and
|
|
FP8 MLA prefill is enabled (auto-detected via
|
|
``_fp8_mla_prefill_supported()``).
|
|
"""
|
|
from aiter import get_ps_metadata_v1
|
|
|
|
prefill = metadata.prefill
|
|
# Caller (build()) only invokes this when prefill tokens exist, so
|
|
# metadata.prefill is guaranteed non-None. Assert to narrow for mypy.
|
|
assert prefill is not None
|
|
qo_indptr = prefill.query_start_loc
|
|
kv_indptr = qo_indptr # new tokens: KV length == Q length
|
|
|
|
# Reuse the existing CPU view of query_start_loc instead of forcing a
|
|
# device->host copy. Prefill batches sit at the tail of the request
|
|
# list, so we slice from num_decodes onwards and rebase to zero, the
|
|
# same transform the parent build applies on device tensors.
|
|
num_decodes = metadata.num_decodes
|
|
qsl_cpu = common_attn_metadata.query_start_loc_cpu
|
|
qo_indptr_cpu = (qsl_cpu[num_decodes:] - qsl_cpu[num_decodes]).to(torch.int32)
|
|
kv_indptr_cpu = qo_indptr_cpu.clone()
|
|
seq_lens_cpu = (qo_indptr_cpu[1:] - qo_indptr_cpu[:-1]).to(torch.int32)
|
|
|
|
# Head counts that are not a multiple of 16 (K3: 12/rank at TP8) are
|
|
# replicate-padded up to one in _mla_fp8_prefill_attn; build the PS
|
|
# metadata for that same padded count so the work/reduce maps and the
|
|
# scratch reservations describe the width the kernel is handed.
|
|
#
|
|
# This was previously max(16, num_heads), which agrees with the helper
|
|
# at every head count reachable today (12 and 16 both give 16) and
|
|
# differs only above 16: at 24 it leaves num_head_k=24 while the
|
|
# forward pads to 32. That is not a correctness bug -- the 24-wide work
|
|
# maps still cover head-tiles 0..23, which are the real heads -- but it
|
|
# is expensive, because a lower head alignment yields more partial
|
|
# tiles: gcd-driven, 24 heads -> 64 tiles vs 32 heads -> 16, i.e.
|
|
# 193.6 MiB of reservations instead of 68.6 MiB at batch=1/qlen=512
|
|
# (measured on gfx950). Sizing both from one helper keeps the widths
|
|
# equal and takes the cheaper tiling.
|
|
num_head_k = AiterMLAHelper.get_fp8_prefill_num_heads(self.num_heads)
|
|
# gqa_ratio = 1
|
|
# qhead_granularity = max(gqa_ratio, 1)
|
|
# qlen_granularity = _FP8_PREFILL_TILE_Q // qhead_granularity
|
|
gqa_ratio = 1
|
|
qhead_granularity = 1
|
|
qlen_granularity = _FP8_PREFILL_TILE_Q
|
|
kvlen_granularity = 128
|
|
block_size = 1 # non-paged: each "page" is one token
|
|
|
|
slot = self._fp8_ps_slot
|
|
self._fp8_ps_slot ^= 1
|
|
staging_free = self._fp8_ps_staging_free[slot]
|
|
with gpu_sync_allowed():
|
|
staging_free.synchronize()
|
|
(
|
|
work_indptr_host,
|
|
work_info_host,
|
|
reduce_indptr_host,
|
|
reduce_final_map_host,
|
|
reduce_partial_map_host,
|
|
) = self._fp8_ps_staging[slot]
|
|
|
|
# work_metadata is not written by the host planner; it stays on device.
|
|
get_ps_metadata_v1(
|
|
qo_indptr_cpu,
|
|
kv_indptr_cpu,
|
|
seq_lens_cpu,
|
|
gqa_ratio,
|
|
num_head_k,
|
|
self.fp8_ps_work_metadata,
|
|
work_indptr_host,
|
|
work_info_host,
|
|
reduce_indptr_host,
|
|
reduce_final_map_host,
|
|
reduce_partial_map_host,
|
|
qhead_granularity=qhead_granularity,
|
|
qlen_granularity=qlen_granularity,
|
|
kvlen_granularity=kvlen_granularity,
|
|
block_size=block_size,
|
|
is_causal=True,
|
|
)
|
|
for dst, src in zip(self._fp8_ps_device_outputs, self._fp8_ps_staging[slot]):
|
|
dst.copy_(src, non_blocking=True)
|
|
staging_free.record()
|
|
|
|
total_prefill_tokens = int(qo_indptr_cpu[-1].item())
|
|
kv_indices = torch.arange(
|
|
total_prefill_tokens, device=qo_indptr.device, dtype=torch.int32
|
|
)
|
|
|
|
# The actual number of active partial tiles for this batch is the
|
|
# final value of reduce_indptr, read from the host plan (no GPU sync).
|
|
num_partial_tiles = int(reduce_indptr_host[-1])
|
|
|
|
# Attach PS metadata to the metadata object so forward_mha can read it.
|
|
metadata.fp8_prefill_qo_indptr = qo_indptr
|
|
metadata.fp8_prefill_kv_indptr = kv_indptr
|
|
metadata.fp8_prefill_kv_indices = kv_indices
|
|
metadata.fp8_prefill_work_indptr = self.fp8_ps_work_indptr
|
|
metadata.fp8_prefill_work_info_set = self.fp8_ps_work_info
|
|
metadata.fp8_prefill_reduce_indptr = self.fp8_ps_reduce_indptr
|
|
metadata.fp8_prefill_reduce_final_map = self.fp8_ps_reduce_final_map
|
|
metadata.fp8_prefill_reduce_partial_map = self.fp8_ps_reduce_partial_map
|
|
metadata.fp8_prefill_max_q_len = prefill.max_query_len
|
|
metadata.fp8_prefill_num_partial_tiles = num_partial_tiles
|
|
|
|
def _build_dcp_verify_row_view(
|
|
self,
|
|
qlen: int,
|
|
block_table: torch.Tensor,
|
|
dcp_tot_seq_lens: torch.Tensor,
|
|
) -> AiterMLADCPVerifyMetadata:
|
|
"""Build one paged-KV row per verify token for DCP verification.
|
|
|
|
Every row is a single query (``qo_indptr`` is an arange), so the whole
|
|
causal structure is carried by ``dcp_local_verify_row_lens`` and the
|
|
kernel applies no tail of its own.
|
|
"""
|
|
assert self.dcp_world_size > 1
|
|
num_reqs = dcp_tot_seq_lens.numel()
|
|
row_lens = AiterMLAHelper.dcp_local_verify_row_lens(
|
|
dcp_tot_seq_lens,
|
|
qlen,
|
|
self.dcp_world_size,
|
|
self.dcp_rank,
|
|
self.cp_kv_cache_interleave_size,
|
|
)
|
|
num_rows = row_lens.numel()
|
|
buffers = self._dcp_verify_buffers
|
|
# Persistent buffers exist exactly when full graphs are on, and they
|
|
# already carry the static bound they were sized for.
|
|
max_kv_seq_len = (
|
|
buffers.max_kv_seq_len
|
|
if buffers is not None
|
|
else max(1, int(row_lens.max().item()))
|
|
)
|
|
page_size = self._dcp_verify_page_size
|
|
max_local_pages = cdiv(max_kv_seq_len, page_size)
|
|
if buffers is not None:
|
|
row_block_table = buffers.block_table[:num_rows]
|
|
buffers.row_lens[:num_rows].copy_(row_lens, non_blocking=True)
|
|
row_lens = buffers.row_lens[:num_rows]
|
|
qo_indptr = buffers.qo_indptr[: num_rows + 1]
|
|
else:
|
|
row_block_table = torch.empty(
|
|
(num_rows, max_local_pages),
|
|
dtype=torch.int32,
|
|
device=block_table.device,
|
|
)
|
|
qo_indptr = torch.arange(
|
|
num_rows + 1,
|
|
dtype=torch.int32,
|
|
device=block_table.device,
|
|
)
|
|
self._fill_dcp_verify_page_table(
|
|
row_block_table, block_table, num_reqs, qlen, page_size
|
|
)
|
|
return AiterMLADCPVerifyMetadata(
|
|
row_lens=row_lens,
|
|
block_table=row_block_table,
|
|
qo_indptr=qo_indptr,
|
|
page_size=page_size,
|
|
max_kv_seq_len=max_kv_seq_len,
|
|
)
|
|
|
|
def _fill_dcp_verify_page_table(
|
|
self,
|
|
row_block_table: torch.Tensor,
|
|
block_table: torch.Tensor,
|
|
num_reqs: int,
|
|
qlen: int,
|
|
page_size: int,
|
|
) -> None:
|
|
"""Fill one page list per request and broadcast it across qlen rows.
|
|
|
|
``page_size`` is the unit the kernel indexes. Segmented MLA uses TILE
|
|
subpages; generic Triton uses the physical KV block. When those are
|
|
equal, pages_per_block is 1 and this copies block IDs as-is.
|
|
|
|
Every row of a request shares the request's shard, so the page list is
|
|
built once per request and repeated; only ``row_lens`` distinguishes the
|
|
rows. A DCP rank holds ``1/dcp_world_size`` of the sequence, so the
|
|
request's block table is always wider than the pages a row can reach.
|
|
"""
|
|
kernel_block_size = self.kernel_block_size
|
|
assert kernel_block_size is not None
|
|
assert page_size > 0
|
|
assert kernel_block_size % page_size == 0, (
|
|
f"DCP verify page_size {page_size} must divide kernel_block_size "
|
|
f"{kernel_block_size}"
|
|
)
|
|
pages_per_block = kernel_block_size // page_size
|
|
max_local_pages = row_block_table.shape[1]
|
|
max_local_blocks = cdiv(max_local_pages, pages_per_block)
|
|
assert max_local_blocks <= block_table.shape[1], (
|
|
f"DCP verify needs {max_local_blocks} blocks per request but the "
|
|
f"block table only has {block_table.shape[1]}"
|
|
)
|
|
if pages_per_block == 1:
|
|
per_req_page_table = block_table[:num_reqs, :max_local_pages]
|
|
else:
|
|
subpage_offsets = torch.arange(
|
|
pages_per_block,
|
|
dtype=block_table.dtype,
|
|
device=block_table.device,
|
|
)
|
|
per_req_page_table = (
|
|
block_table[:num_reqs, :max_local_blocks, None] * pages_per_block
|
|
+ subpage_offsets
|
|
).flatten(1)[:, :max_local_pages]
|
|
# Broadcast the request's page list across its rows instead of
|
|
# materializing a repeat_interleave copy.
|
|
row_block_table.unflatten(0, (num_reqs, qlen)).copy_(
|
|
per_req_page_table.unsqueeze(1)
|
|
)
|
|
|
|
def _build_decode(
|
|
self,
|
|
block_table_tensor: torch.Tensor,
|
|
seq_lens_device: torch.Tensor,
|
|
max_seq_len: int,
|
|
query_start_loc_cpu: torch.Tensor,
|
|
query_start_loc_device: torch.Tensor,
|
|
num_decode_tokens: int,
|
|
max_query_len: int,
|
|
dcp_tot_seq_lens_device: torch.Tensor | None,
|
|
) -> AiterMLADecodeMetadata:
|
|
causal = self._decode_causal
|
|
device = self.device
|
|
num_reqs = seq_lens_device.size(0)
|
|
qo_len = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
|
varlen_decode = self._varlen_decode_qlen > 0 and causal
|
|
if varlen_decode:
|
|
max_qo_len = self._varlen_decode_qlen
|
|
pad_uniform_mtp = True
|
|
else:
|
|
max_qo_len = qo_len.max().item()
|
|
padded_mtp_qo_len = self._uniform_padded_mtp_qo_len(
|
|
qo_len, max_qo_len, num_decode_tokens
|
|
)
|
|
if padded_mtp_qo_len > 0:
|
|
max_qo_len = padded_mtp_qo_len
|
|
pad_uniform_mtp = padded_mtp_qo_len > 0
|
|
|
|
seq_lens_for_kernel = seq_lens_device
|
|
# the global lengths need the same dummy-row
|
|
# treatment as the local ones below, but only for g_kv_indptr -- the
|
|
# metadata field keeps the true values, which is what the segmented
|
|
# route wants (it derives its own row lens and gives a dummy row
|
|
# length 0, correctly).
|
|
g_tot_seq_lens = dcp_tot_seq_lens_device
|
|
num_kernel_reqs = num_reqs
|
|
if pad_uniform_mtp:
|
|
qo_lens_device = (
|
|
query_start_loc_device[1 : num_reqs + 1]
|
|
- query_start_loc_device[:num_reqs]
|
|
).to(torch.int32)
|
|
seq_lens_for_kernel = torch.where(
|
|
qo_lens_device > 0,
|
|
seq_lens_for_kernel,
|
|
seq_lens_for_kernel.new_full((), max_qo_len),
|
|
)
|
|
# A dummy row just got a LOCAL length of max_qo_len, so its GLOBAL
|
|
# length has to say the same thing or the cprr mask is degenerate:
|
|
# global_len 0 puts the row query positions at {-3,-2,-1} while its
|
|
# local shard sits at {rank, W+rank, 2W+rank}, so every entry is
|
|
# masked and the row softmaxes over nothing. max_qo_len * W is
|
|
# exactly the global length whose round-robin shard is max_qo_len
|
|
# on every rank.
|
|
if g_tot_seq_lens is not None:
|
|
g_tot_seq_lens = torch.where(
|
|
qo_lens_device > 0,
|
|
g_tot_seq_lens,
|
|
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
|
|
)
|
|
|
|
varlen_q_rows = varlen_o_rows = None
|
|
if varlen_decode:
|
|
seq_lens_for_kernel = seq_lens_device + (max_qo_len - qo_lens_device)
|
|
varlen_q_rows, varlen_o_rows = self._fill_varlen_rows(
|
|
query_start_loc_device, qo_lens_device, num_reqs
|
|
)
|
|
|
|
if self._graph_seq_lens is not None:
|
|
self._graph_seq_lens[:num_kernel_reqs].copy_(
|
|
seq_lens_for_kernel, non_blocking=True
|
|
)
|
|
seq_lens_for_kernel = self._graph_seq_lens[:num_kernel_reqs]
|
|
|
|
# The aiter kernel always operates with page_size=1 (the wrapper
|
|
# flattens kv_buffer). last_page_len is always 1.
|
|
paged_kv_last_page_len = self.paged_kv_last_page_len[:num_kernel_reqs]
|
|
|
|
# indptr: cumsum of seq_lens (one page per token in the flat view)
|
|
paged_kv_indptr = torch.cat(
|
|
[
|
|
torch.zeros(1, dtype=torch.int32, device=device),
|
|
seq_lens_for_kernel.cumsum(dim=0, dtype=torch.int32),
|
|
]
|
|
)
|
|
use_gluon_decode = AiterMLAHelper.use_gluon_decode(
|
|
self._decode_num_heads,
|
|
int(max_qo_len),
|
|
self._kv_cache_dtype_str,
|
|
self._kv_cache_bytes,
|
|
)
|
|
use_gluon_verify = AiterMLAHelper.use_gluon_verify(
|
|
self._decode_num_heads,
|
|
int(max_qo_len),
|
|
self._kv_cache_dtype_str,
|
|
self.dcp_world_size,
|
|
causal,
|
|
self._kv_cache_bytes,
|
|
)
|
|
dcp_route = _DCPDecodeRoute.PLAIN
|
|
if self.dcp_world_size > 1:
|
|
dcp_route = _select_dcp_decode_route(
|
|
supports_segmented=self._supports_segmented_dcp_verify,
|
|
supports_triton=self._use_triton_dcp_verify,
|
|
causal=causal,
|
|
max_qo_len=int(max_qo_len),
|
|
asm_selected=self._asm_dcp_verify,
|
|
)
|
|
|
|
# Row-view DCP verify carries its own per-row page table, so the
|
|
# flat per-token view is dead work for it. Leave the buffer alone and
|
|
# hand the metadata None, so a future reader cannot pick up whatever
|
|
# the previous batch left behind.
|
|
paged_kv_indices = None
|
|
if not _is_row_view_dcp_route(dcp_route):
|
|
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
|
|
self.paged_kv_indices.fill_(-1)
|
|
|
|
# Expand block_table entries into per-token flat indices.
|
|
# When kernel_block_size=1, this degrades to a direct copy (identical
|
|
# to the original _copy_page_indices_kernel).
|
|
# When kernel_block_size=K>1, block_table entry b covering K tokens
|
|
# gets expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
|
|
# Chunk count comes from the block table width, an upper bound on
|
|
# tokens per request that is available host-side, so this adds no
|
|
# device synchronisation. Programs whose chunk lies past
|
|
# num_tokens mask out entirely.
|
|
max_tokens_per_req = block_table_tensor.shape[1] * self.kernel_block_size
|
|
if varlen_decode:
|
|
max_tokens_per_req += max_qo_len - 1
|
|
num_chunks = max(1, cdiv(max_tokens_per_req, 1024))
|
|
_expand_page_indices_kernel[(num_reqs, num_chunks)](
|
|
self.paged_kv_indices,
|
|
block_table_tensor,
|
|
block_table_tensor.stride(0),
|
|
block_table_tensor.stride(1),
|
|
paged_kv_indptr,
|
|
seq_lens_device,
|
|
KERNEL_BLOCK_SIZE=self.kernel_block_size,
|
|
BLOCK_SIZE=1024,
|
|
CLAMP_TAIL=varlen_decode,
|
|
)
|
|
paged_kv_indices = self.paged_kv_indices
|
|
|
|
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
|
|
self.paged_kv_indptr[: 1 + num_kernel_reqs].copy_(
|
|
paged_kv_indptr, non_blocking=True
|
|
)
|
|
self.paged_kv_indptr[1 + num_kernel_reqs :].fill_(paged_kv_indptr[-1])
|
|
paged_kv_indptr = self.paged_kv_indptr[: 1 + num_kernel_reqs]
|
|
|
|
# paged_kv_last_page_len already uses the pre-initialized buffer slice
|
|
# (set above), so no copy needed - buffer is always 1s.
|
|
|
|
if pad_uniform_mtp:
|
|
qo_indptr_src = AiterMLAHelper.qo_indptr_for_uniform_qlen(
|
|
num_kernel_reqs, int(max_qo_len), device
|
|
)
|
|
else:
|
|
qo_indptr_src = query_start_loc_device[: 1 + num_kernel_reqs]
|
|
self.qo_indptr[: 1 + num_kernel_reqs].copy_(
|
|
qo_indptr_src, non_blocking=True
|
|
)
|
|
self.qo_indptr[1 + num_kernel_reqs :] = qo_indptr_src[-1]
|
|
qo_indptr = self.qo_indptr[: 1 + num_kernel_reqs]
|
|
|
|
else:
|
|
if max_qo_len == 1:
|
|
qo_indptr = AiterMLAHelper.qo_indptr_for_uniform_qlen(
|
|
num_kernel_reqs, 1, device
|
|
)
|
|
else:
|
|
if pad_uniform_mtp:
|
|
qo_indptr = AiterMLAHelper.qo_indptr_for_uniform_qlen(
|
|
num_kernel_reqs, int(max_qo_len), device
|
|
)
|
|
else:
|
|
qo_indptr = query_start_loc_device[: 1 + num_kernel_reqs]
|
|
|
|
g_kv_indptr = None
|
|
if dcp_route is _DCPDecodeRoute.CPRR and g_tot_seq_lens is not None:
|
|
assert self._g_kv_indptr_buf is not None
|
|
_ngk = g_tot_seq_lens.shape[0]
|
|
g_kv_indptr = self._g_kv_indptr_buf[: _ngk + 1]
|
|
g_kv_indptr[1:].copy_(g_tot_seq_lens.cumsum(dim=0, dtype=torch.int32))
|
|
# A FULL cudagraph replays at its capture-time request count, so a
|
|
# smaller real batch leaves the tail holding the PREVIOUS step's
|
|
# cumsum. Flatten it the way paged_kv_indptr is flattened below, so
|
|
# padding rows read global_len == 0 and agree with their zero-length
|
|
# local shard instead of a stale global length.
|
|
self._g_kv_indptr_buf[_ngk + 1 :].fill_(g_kv_indptr[-1])
|
|
|
|
has_persistent_metadata = False
|
|
# Only the asm decode consumes the schedule, so gate on the routing
|
|
# rather than on num_heads >= 16, which denies it to a padded rank
|
|
# running the same asm kernels. Gluon and the row-view DCP routes
|
|
# (segmented and gfx942 Triton) cover the non-ASM decode/verify paths.
|
|
use_persistent_metadata = (
|
|
not use_gluon_decode
|
|
and not use_gluon_verify
|
|
and not _is_row_view_dcp_route(dcp_route)
|
|
# A padded rank has no bf16 persistent kernel past qlen 4 where the
|
|
# gfx950 fold is absent; the non-persistent entry covers it. fp8
|
|
# keeps the schedule -- its fold rejects non-persistent outright.
|
|
# A non-causal block keeps it too: what the fold drops past qlen 4
|
|
# is the block's causal staircase, which a non-causal block does
|
|
# not have, and the schedule is the only thing carrying its mask.
|
|
and (
|
|
not causal
|
|
or self._decode_num_heads >= AiterMLAHelper._AITER_MIN_MLA_HEADS
|
|
or max_qo_len <= AiterMLAHelper._ASM_PADDED_MAX_PS_QLEN
|
|
or is_quantized_kv_cache(self._kv_cache_dtype_str)
|
|
)
|
|
and max_qo_len >= 1
|
|
and max_qo_len <= self._mtp_decode_qlen
|
|
)
|
|
if (
|
|
not causal
|
|
and max_qo_len == 2
|
|
and is_quantized_kv_cache(self._kv_cache_dtype_str)
|
|
and not AiterMLAHelper.has_fp8_non_causal_qlen2_kernel(
|
|
self._decode_num_heads
|
|
)
|
|
):
|
|
# AITER's fp8 dispatch folds (gqa 16, qlen 3 or 4) onto the
|
|
# qseqlen-4 kernel but never lists 2. Only 32/64/96/128 heads
|
|
# at qlen 2 fold onto that 16-head / 4-token non-causal kernel;
|
|
# 16-head qlen 2, and padded 48/80/112, keep Q2 and abort the
|
|
# process rather than raising. The bf16 fold has no such hole.
|
|
raise ValueError(
|
|
"AITER has no non-causal fp8 MLA kernel for this 2-token "
|
|
"query block. Pin the draft to TRITON_MLA for this "
|
|
"speculative config."
|
|
)
|
|
if use_persistent_metadata:
|
|
from aiter import get_mla_metadata_v1
|
|
|
|
uni_qo_len = (
|
|
max_qo_len if pad_uniform_mtp or torch.all(qo_len == max_qo_len) else -1
|
|
)
|
|
cprr_kwargs: dict = {}
|
|
is_causal = causal
|
|
if self._asm_dcp_verify:
|
|
cprr_kwargs = dict(
|
|
max_split_per_batch=self._mla_max_split_per_batch,
|
|
intra_batch_mode=False,
|
|
)
|
|
if dcp_route is _DCPDecodeRoute.CPRR:
|
|
is_causal = False
|
|
cprr_kwargs["is_cp_round_robin"] = True
|
|
metadata_num_heads = (
|
|
self._asm_dcp_verify_heads
|
|
if dcp_route is _DCPDecodeRoute.CPRR
|
|
else AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
|
|
)
|
|
get_mla_metadata_v1(
|
|
qo_indptr,
|
|
paged_kv_indptr,
|
|
paged_kv_last_page_len,
|
|
metadata_num_heads,
|
|
1,
|
|
is_causal,
|
|
self._mla_work_meta_data,
|
|
self._mla_work_info_set,
|
|
self._mla_work_indptr,
|
|
self._mla_reduce_indptr,
|
|
self._mla_reduce_final_map,
|
|
self._mla_reduce_partial_map,
|
|
page_size=1,
|
|
kv_granularity=16,
|
|
max_seqlen_qo=max_qo_len,
|
|
uni_seqlen_qo=uni_qo_len,
|
|
fast_mode=True,
|
|
dtype_q=self._mla_q_dtype,
|
|
dtype_kv=self._mla_kv_dtype,
|
|
**cprr_kwargs,
|
|
)
|
|
has_persistent_metadata = True
|
|
|
|
# Small-head multi-token verify uses mla_gluon's 4-D MTP entry over the
|
|
# ordinary per-request paged-KV view, so there is no expanded per-token
|
|
# buffer to build here. mla_gluon still wants a lower bound on the KV
|
|
# length it is asked to split; on the verify path that bound is over
|
|
# active requests, not cudagraph padding rows pinned to max_qo_len.
|
|
# The .item() below runs in the builder, outside the captured region,
|
|
# so it does not abort HIP graph capture the way the per-layer syncs
|
|
# in forward_mqa did. DCP verify never reaches here -- it carries its
|
|
# own per-row lengths instead of a single split bound.
|
|
min_kv_seq_len = 1
|
|
if use_gluon_verify:
|
|
per_req_len = paged_kv_indptr[1:] - paged_kv_indptr[:-1]
|
|
if pad_uniform_mtp:
|
|
active = qo_lens_device > 0
|
|
if active.any():
|
|
min_kv_seq_len = int(per_req_len[active].min().item())
|
|
else:
|
|
min_kv_seq_len = int(per_req_len.min().item())
|
|
|
|
dcp_verify = None
|
|
if _is_row_view_dcp_route(dcp_route):
|
|
assert causal
|
|
assert dcp_tot_seq_lens_device is not None
|
|
dcp_verify = self._build_dcp_verify_row_view(
|
|
int(max_qo_len),
|
|
block_table_tensor,
|
|
dcp_tot_seq_lens_device,
|
|
)
|
|
|
|
attn_metadata = AiterMLADecodeMetadata(
|
|
block_table=block_table_tensor,
|
|
seq_lens=seq_lens_for_kernel,
|
|
paged_kv_indptr=paged_kv_indptr,
|
|
paged_kv_indices=paged_kv_indices,
|
|
paged_kv_last_page_len=paged_kv_last_page_len,
|
|
qo_indptr=qo_indptr,
|
|
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
|
|
max_qo_len=max_qo_len,
|
|
min_kv_seq_len=min_kv_seq_len,
|
|
dcp_route=dcp_route,
|
|
dcp_verify=dcp_verify,
|
|
g_kv_indptr=g_kv_indptr,
|
|
cp_world_size=self.dcp_world_size,
|
|
cp_rank=self.dcp_rank,
|
|
asm_decode_num_heads=(
|
|
self._asm_dcp_verify_heads if dcp_route is _DCPDecodeRoute.CPRR else 0
|
|
),
|
|
mla_num_kv_splits=(
|
|
self._mla_max_split_per_batch
|
|
if dcp_route is _DCPDecodeRoute.CPRR
|
|
else 0
|
|
),
|
|
use_gluon_decode=use_gluon_decode,
|
|
use_gluon_verify=use_gluon_verify,
|
|
attn_out_dtype=self.decode_attn_out_dtype,
|
|
has_persistent_metadata=has_persistent_metadata,
|
|
varlen_q_rows=varlen_q_rows,
|
|
varlen_o_rows=varlen_o_rows,
|
|
)
|
|
|
|
return attn_metadata
|
|
|
|
def _fill_varlen_rows(
|
|
self,
|
|
query_start_loc_device: torch.Tensor,
|
|
qo_lens_device: torch.Tensor,
|
|
num_reqs: int,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
width = self._varlen_decode_qlen
|
|
num_rows = num_reqs * width
|
|
row_req = self._varlen_row_req[:num_rows]
|
|
row_pos = self._varlen_row_pos[:num_rows]
|
|
row_qlen = qo_lens_device.to(torch.int64).index_select(0, row_req)
|
|
row_start = query_start_loc_device.to(torch.int64).index_select(0, row_req)
|
|
is_real = row_pos < row_qlen
|
|
packed_row = row_start + row_pos
|
|
q_rows = self._varlen_q_rows[:num_rows]
|
|
q_rows.copy_(torch.where(is_real, packed_row, 0))
|
|
o_rows = self._varlen_o_rows
|
|
o_rows.zero_()
|
|
dump_slot = o_rows.size(0) - 1
|
|
o_rows.scatter_(
|
|
0,
|
|
torch.where(is_real, packed_row, dump_slot),
|
|
self._varlen_row_ids[:num_rows],
|
|
)
|
|
return q_rows, o_rows
|
|
|
|
def build(
|
|
self,
|
|
common_prefix_len: int,
|
|
common_attn_metadata: CommonAttentionMetadata,
|
|
fast_build: bool = False,
|
|
) -> AiterMLAMetadata:
|
|
# The common MLA builder owns the causal flag but its _build_decode
|
|
# contract is shared with CUDA backends. Keep this ROCm-specific state
|
|
# on the per-ubatch Aiter builder instead of changing that shared API;
|
|
# super().build invokes _build_decode synchronously.
|
|
self._decode_causal = common_attn_metadata.causal
|
|
attn_metadata = super().build(
|
|
common_prefix_len, common_attn_metadata, fast_build
|
|
)
|
|
prefill = attn_metadata.prefill
|
|
if (
|
|
prefill is not None
|
|
and prefill.chunked_context is not None
|
|
and self.dcp_world_size > 1
|
|
and self._supports_fused_dcp_prefill
|
|
):
|
|
attn_metadata.dcp_context_row_indices = [
|
|
context_row_indices(chunk, prefill.block_table.device)
|
|
for chunk in prefill.chunked_context.chunks
|
|
]
|
|
if (
|
|
attn_metadata.decode is not None
|
|
and attn_metadata.decode.has_persistent_metadata
|
|
):
|
|
attn_metadata.work_meta_data = self._mla_work_meta_data
|
|
attn_metadata.work_indptr = self._mla_work_indptr
|
|
attn_metadata.work_info_set = self._mla_work_info_set
|
|
attn_metadata.reduce_indptr = self._mla_reduce_indptr
|
|
attn_metadata.reduce_final_map = self._mla_reduce_final_map
|
|
attn_metadata.reduce_partial_map = self._mla_reduce_partial_map
|
|
if (
|
|
self._fp8_prefill_enabled
|
|
and attn_metadata.prefill is not None
|
|
and attn_metadata.prefill.chunked_context is None
|
|
):
|
|
self._build_fp8_prefill_ps_metadata(attn_metadata, common_attn_metadata)
|
|
return attn_metadata
|
|
|
|
|
|
@triton.jit
|
|
def _expand_page_indices_kernel(
|
|
page_indices,
|
|
block_table,
|
|
block_table_stride_0,
|
|
block_table_stride_1,
|
|
cu_num_tokens,
|
|
num_real_tokens,
|
|
KERNEL_BLOCK_SIZE: tl.constexpr,
|
|
BLOCK_SIZE: tl.constexpr,
|
|
CLAMP_TAIL: tl.constexpr = False,
|
|
):
|
|
"""Expand block table entries into per-token flat page indices.
|
|
|
|
The aiter MLA kernel always operates with page_size=1 internally
|
|
(kv_buffer is flattened via .view(-1, 1, 1, H)). This kernel converts
|
|
block-level indices from the block table into individual token positions
|
|
in the flattened KV buffer.
|
|
|
|
When KERNEL_BLOCK_SIZE=1: block_idx=t, offset=0, flat=block_id
|
|
(equivalent to a direct copy -- no regression from the original kernel).
|
|
|
|
When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens)
|
|
is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1).
|
|
|
|
With CLAMP_TAIL, entries past num_real_tokens[req] (padding a request's
|
|
page list out to its kernel length) repeat the request's first page, so
|
|
they never index past its block-table row.
|
|
"""
|
|
# One program per (request, token-chunk). Parallelising over requests
|
|
# alone degenerates at low concurrency: with num_reqs == 1 a single
|
|
# workgroup walked the whole sequence in a serial loop.
|
|
req_idx = tl.program_id(0)
|
|
chunk_idx = tl.program_id(1)
|
|
row_ptr = block_table + req_idx * block_table_stride_0
|
|
start_idx = tl.load(cu_num_tokens + req_idx)
|
|
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
|
|
if CLAMP_TAIL:
|
|
real_tokens = tl.load(num_real_tokens + req_idx)
|
|
|
|
# The grid is sized from the block table width, an upper bound over all
|
|
# requests, so a ragged batch launches chunks past a short request's end.
|
|
# Returning here keeps those programs from issuing masked-out loads and
|
|
# stores at all.
|
|
chunk_start = chunk_idx * BLOCK_SIZE
|
|
if chunk_start >= num_tokens:
|
|
return
|
|
|
|
token_offsets = chunk_start + tl.arange(0, BLOCK_SIZE)
|
|
mask = token_offsets < num_tokens
|
|
source_offsets = token_offsets
|
|
if CLAMP_TAIL:
|
|
source_offsets = tl.where(token_offsets < real_tokens, token_offsets, 0)
|
|
|
|
# Which block in the block table does this token belong to?
|
|
block_idx = source_offsets // KERNEL_BLOCK_SIZE
|
|
# Offset within that block
|
|
offset_in_block = source_offsets % KERNEL_BLOCK_SIZE
|
|
|
|
# Load the block ID from the block table
|
|
# Both strides are taken from the caller: the block table is a view owned
|
|
# elsewhere, so a unit column stride must not be assumed.
|
|
block_ids = tl.load(row_ptr + block_idx * block_table_stride_1, mask=mask)
|
|
|
|
# Compute flat index in the flattened kv_buffer
|
|
flat_indices = block_ids * KERNEL_BLOCK_SIZE + offset_in_block
|
|
|
|
tl.store(
|
|
page_indices + start_idx + token_offsets,
|
|
flat_indices,
|
|
mask=mask,
|
|
)
|
|
|
|
|
|
class AiterMLAHelper:
|
|
"""AITER MLA persistent (asm) decode requires a multiple of 16 heads. Unaligned
|
|
head counts through 128 are padded to the next multiple of 16 by tiling the
|
|
query heads and slicing to the padded size. Native H24 AITER builds bypass
|
|
that padding. Small divisors of 16 retain the existing repeat_interleave and
|
|
strided-unpad behavior. Native and aligned counts pass through without
|
|
copies.
|
|
"""
|
|
|
|
_AITER_MIN_MLA_HEADS: Final = 16
|
|
_AITER_MAX_PADDED_MLA_HEADS: Final = 128
|
|
# Largest qlen the padded gqa=16 asm decode has a bf16 persistent kernel
|
|
# for. Above it only the non-persistent qseqlen=8 entry exists, and the
|
|
# fold that reaches a persistent one is gfx950-only.
|
|
_ASM_PADDED_MAX_PS_QLEN: Final = 4
|
|
_AITER_UNSUPPORTED_HEADS: ClassVar[tuple[int, ...]] = ()
|
|
# Pinned AITER v0.1.21.post2 folds these fp8 qlen-2 counts onto the
|
|
# non-causal 16-head / 4-token kernel. Other multiples of 16 (48/80/112)
|
|
# fold to H16 while keeping Q2, which has no non-causal fp8 entry.
|
|
_AITER_FP8_NON_CAUSAL_QLEN2_HEADS: ClassVar[tuple[int, ...]] = (32, 64, 96, 128)
|
|
|
|
@staticmethod
|
|
def qo_indptr_for_uniform_qlen(
|
|
num_reqs: int,
|
|
qlen: int,
|
|
device: torch.device,
|
|
dtype: torch.dtype = torch.int32,
|
|
) -> torch.Tensor:
|
|
"""Build ``[0, qlen, 2*qlen, ..., num_reqs*qlen]``."""
|
|
return torch.arange(
|
|
0,
|
|
(num_reqs + 1) * qlen,
|
|
step=qlen,
|
|
dtype=dtype,
|
|
device=device,
|
|
)
|
|
|
|
@staticmethod
|
|
def check_num_heads_validity(num_heads: int):
|
|
assert AiterMLAHelper.is_valid_num_heads(num_heads), (
|
|
"ROCM AITER MLA requires a positive multiple of 16 heads, or an "
|
|
"unaligned head count up to 128 (padded to the next multiple of "
|
|
f"16), but got {num_heads}.\n"
|
|
f"Try adjusting tensor_parallel_size value."
|
|
)
|
|
|
|
@staticmethod
|
|
def is_valid_num_heads(num_heads: int) -> bool:
|
|
return (
|
|
num_heads > 0
|
|
and num_heads not in AiterMLAHelper._AITER_UNSUPPORTED_HEADS
|
|
and (
|
|
num_heads <= AiterMLAHelper._AITER_MAX_PADDED_MLA_HEADS
|
|
or num_heads % AiterMLAHelper._AITER_MIN_MLA_HEADS == 0
|
|
)
|
|
)
|
|
|
|
@staticmethod
|
|
def has_fp8_non_causal_qlen2_kernel(num_heads: int) -> bool:
|
|
"""Whether fp8 (num_heads, qlen=2) folds onto a non-causal ASM kernel.
|
|
|
|
Pinned AITER v0.1.21.post2 folds 32/64/96/128 heads at qlen 2 onto the
|
|
non-causal 16-head / 4-token kernel. A 16-head (or padded-to-16) qlen-2
|
|
block has no matching entry, and padded 48/80/112 keep Q2 after the
|
|
H16 fold.
|
|
"""
|
|
kernel_heads = AiterMLAHelper.get_actual_mla_num_heads(num_heads)
|
|
return kernel_heads in AiterMLAHelper._AITER_FP8_NON_CAUSAL_QLEN2_HEADS
|
|
|
|
@staticmethod
|
|
def get_actual_mla_num_heads(num_heads: int) -> int:
|
|
if num_heads == 24 and _aiter_mla_native_h24_supported():
|
|
return num_heads
|
|
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
|
|
return -(-num_heads // m) * m
|
|
|
|
@staticmethod
|
|
def get_fp8_prefill_num_heads(num_heads: int) -> int:
|
|
"""Head count the FP8 PS asm prefill runs at: the next multiple of 16.
|
|
|
|
Deliberately *not* ``get_actual_mla_num_heads``. That one carves out
|
|
native H24 when ``_aiter_mla_native_h24_supported()``, which probes the
|
|
asm *decode* reducer and metadata. The prefill is a different kernel
|
|
pair (``mla_prefill_ps_asm_fwd`` + ``mla_reduce_v1``) with no such
|
|
probe, so 24 heads pad to 32 here even on a native-H24 build.
|
|
|
|
The PS metadata in ``_init_fp8_prefill_ps_buffers``/``build()`` must be
|
|
sized with this same function, or the work/reduce maps describe a
|
|
different head count than the kernel is handed.
|
|
|
|
This function itself has no upper bound; the ceiling comes from the
|
|
``is_valid_num_heads`` gate, which rejects counts above
|
|
``_AITER_MAX_PADDED_MLA_HEADS`` (128) unless they are already
|
|
16-aligned. That bound was established for the asm *decode* padding, so
|
|
an architecture with, say, 136 heads per rank would be refused the
|
|
prefill here for a reason that was never measured against this kernel
|
|
pair. Revisit the constant rather than special-casing prefill.
|
|
"""
|
|
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
|
|
return -(-num_heads // m) * m
|
|
|
|
@staticmethod
|
|
def get_mla_padded_q(
|
|
num_heads: int, q: torch.Tensor, target_heads: int | None = None
|
|
) -> torch.Tensor:
|
|
m = (
|
|
target_heads
|
|
if target_heads is not None
|
|
else AiterMLAHelper.get_actual_mla_num_heads(num_heads)
|
|
)
|
|
if num_heads == m:
|
|
return q
|
|
if m % num_heads == 0:
|
|
return q.repeat_interleave(m // num_heads, dim=1)
|
|
# Non-divisor head counts cannot be padded by repeat_interleave. Tile
|
|
# the query heads and slice to exactly m. MLA attention is independent
|
|
# per query head over the shared KV, so padding heads cannot affect
|
|
# heads [0:num_heads]; they are sliced back off the output.
|
|
reps = -(-m // num_heads) # ceil(m / num_heads)
|
|
# Slicing a tiled tensor yields a non-contiguous view. The asm decode
|
|
# reads q as packed [tokens, m, head_dim], so materialize it.
|
|
return q.repeat(1, reps, 1)[:, :m, :].contiguous()
|
|
|
|
@staticmethod
|
|
def get_mla_unpadded_o(num_heads: int, o: torch.Tensor) -> torch.Tensor:
|
|
return AiterMLAHelper._get_mla_unpadded_heads(num_heads, o)
|
|
|
|
@staticmethod
|
|
def _get_mla_unpadded_heads(num_heads: int, tensor: torch.Tensor) -> torch.Tensor:
|
|
m = AiterMLAHelper.get_actual_mla_num_heads(num_heads)
|
|
if num_heads == m:
|
|
return tensor
|
|
if m % num_heads == 0:
|
|
return tensor[:, :: m // num_heads, ...]
|
|
# Undo the tile-padding from get_mla_padded_q: the real heads are the
|
|
# first num_heads.
|
|
return tensor[:, :num_heads, ...]
|
|
|
|
@staticmethod
|
|
def get_mla_unpadded_lse(num_heads: int, lse: torch.Tensor) -> torch.Tensor:
|
|
return AiterMLAHelper._get_mla_unpadded_heads(num_heads, lse)
|
|
|
|
@staticmethod
|
|
def use_gluon_decode(
|
|
num_heads: int,
|
|
max_qo_len: int,
|
|
kv_cache_dtype: str,
|
|
kv_cache_bytes: int | None = None,
|
|
) -> bool:
|
|
# Small-head (<16) single-token decode takes either the Gluon kernel or
|
|
# the padded asm persistent decode, selected by
|
|
# VLLM_ROCM_AITER_MLA_ASM_PADDING and the arch (Gluon is gfx950 only).
|
|
m = AiterMLAHelper._AITER_MIN_MLA_HEADS
|
|
if num_heads >= m or max_qo_len != 1:
|
|
return False
|
|
# Gluon's only fp8-KV regime, bh16bn128, is a bf16-query kernel with a
|
|
# hardcoded scale that asserts batch_size == 1, so it cannot serve a
|
|
# decode batch. Checked before the mode knob: an explicit "gluon"
|
|
# request under fp8 would assert immediately.
|
|
if is_quantized_kv_cache(kv_cache_dtype):
|
|
return False
|
|
mode = _aiter_mla_small_head_mode()
|
|
if mode == "asm":
|
|
return False
|
|
if not _gluon_mla_decode_supported():
|
|
return False
|
|
if mode != "gluon" and m % num_heads != 0:
|
|
return False
|
|
# Last, so the size warning only fires where Gluon would have run.
|
|
return _gluon_kv_cache_in_bounds(kv_cache_bytes)
|
|
|
|
@staticmethod
|
|
def use_gluon_verify(
|
|
num_heads: int,
|
|
max_qo_len: int,
|
|
kv_cache_dtype: str,
|
|
dcp_world_size: int = 1,
|
|
causal: bool = True,
|
|
kv_cache_bytes: int | None = None,
|
|
) -> bool:
|
|
"""Whether a small-head multi-token verify uses native Gluon MTP.
|
|
|
|
bf16 has no gqa<16, qseqlen>1 asm kernel, so verify goes through
|
|
``mla_gluon``'s 4-D MTP entry (``q`` shaped ``[batch, qlen, nhead, dim]``)
|
|
with ``use_2d_view=False``. fp8 has one via the q-row fold and must not
|
|
come here: the MTP path hands Gluon the batch size its fp8 regime
|
|
asserts against. A predicate rather than inline in forward_mqa so the
|
|
builder sees the same answer the impl acts on.
|
|
|
|
DCP verify is excluded: its per-row causal windows are served by the
|
|
segmented path, which Gluon's MTP entry cannot express.
|
|
|
|
Gluon masks the block causally with no way to turn it off, so a
|
|
non-causal block takes the padded asm decode whatever the head count
|
|
-- padding is what gives it a kernel there.
|
|
"""
|
|
if not causal:
|
|
return False
|
|
if max_qo_len <= 1 or dcp_world_size > 1:
|
|
return False
|
|
if is_quantized_kv_cache(kv_cache_dtype):
|
|
return False
|
|
if num_heads >= AiterMLAHelper._AITER_MIN_MLA_HEADS:
|
|
return False
|
|
if not _gluon_mla_decode_supported():
|
|
return False
|
|
# Same arch, mode and cache-size gating as use_gluon_decode.
|
|
if _aiter_mla_small_head_mode() == "asm":
|
|
return False
|
|
return _gluon_kv_cache_in_bounds(kv_cache_bytes)
|
|
|
|
@staticmethod
|
|
def dcp_local_verify_row_lens(
|
|
tot_seq_lens: torch.Tensor,
|
|
qlen: int,
|
|
dcp_world_size: int,
|
|
dcp_rank: int,
|
|
cp_interleave: int,
|
|
) -> torch.Tensor:
|
|
"""Local KV length of every verify row, in global-causal order.
|
|
|
|
Row ``t`` of a qlen-token verify attends global positions
|
|
``[0, seq_len - qlen + t]``, its own token included, so its local length
|
|
is the round-robin count evaluated at that bound. Truncating causally in
|
|
global coordinates has to happen before the shard mapping: the tokens a
|
|
row drops sit on ``qlen - 1 - t`` different ranks, while subtracting the
|
|
offset from a request's local length takes one from every rank.
|
|
"""
|
|
offsets = torch.arange(
|
|
1,
|
|
qlen + 1,
|
|
device=tot_seq_lens.device,
|
|
dtype=tot_seq_lens.dtype,
|
|
)
|
|
visible = (tot_seq_lens.unsqueeze(1) - qlen + offsets).clamp_(min=0)
|
|
return get_dcp_local_seq_lens(
|
|
visible,
|
|
dcp_world_size,
|
|
dcp_rank,
|
|
cp_interleave,
|
|
).flatten()
|
|
|
|
|
|
class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
|
# DCP decode paths return natural-log softmax LSE for the cross-rank merge.
|
|
can_return_lse_for_decode: bool = True
|
|
supports_dcp: bool = True
|
|
# Measured on gfx950: aiter.mla.mla_decode_fwd(return_lse=True) matches
|
|
# logsumexp to fp32 exactly, and merge_mla_segments_triton converts AITER's
|
|
# base-2 segment statistics with LOGE2. Stated rather than inherited because
|
|
# the DCP combine silently corrupts the softmax denominator if it disagrees.
|
|
lse_base_on_e: bool = True
|
|
|
|
@property
|
|
def _decode_num_heads(self) -> int:
|
|
"""Return the query-head count after DCP gathering."""
|
|
return self.num_heads * self.dcp_world_size
|
|
|
|
def __init__(
|
|
self,
|
|
num_heads: int,
|
|
head_size: int,
|
|
scale: float,
|
|
num_kv_heads: int,
|
|
alibi_slopes: list[float] | None,
|
|
sliding_window: int | None,
|
|
kv_cache_dtype: str,
|
|
logits_soft_cap: float | None,
|
|
attn_type: str,
|
|
kv_sharing_target_layer_name: str | None,
|
|
# MLA Specific Arguments
|
|
**mla_args,
|
|
) -> None:
|
|
super().__init__(
|
|
num_heads,
|
|
head_size,
|
|
scale,
|
|
num_kv_heads,
|
|
alibi_slopes,
|
|
sliding_window,
|
|
kv_cache_dtype,
|
|
logits_soft_cap,
|
|
attn_type,
|
|
kv_sharing_target_layer_name,
|
|
**mla_args,
|
|
)
|
|
AiterMLAHelper.check_num_heads_validity(num_heads)
|
|
AiterMLAHelper.check_num_heads_validity(self._decode_num_heads)
|
|
self._sm_count = current_platform.num_compute_units()
|
|
|
|
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
|
|
if any(unsupported_features):
|
|
raise NotImplementedError(
|
|
"Aiter MLA does not support one of the following: "
|
|
"alibi_slopes, sliding_window, logits_soft_cap"
|
|
)
|
|
|
|
from aiter import flash_attn_varlen_func
|
|
|
|
self.flash_attn_varlen_func = flash_attn_varlen_func
|
|
|
|
# FP8 MLA prefill kernel imports (lazy, only when enabled).
|
|
# Auto-enabled on gfx950 when AITER ships the kernels. Only runs when the
|
|
# KV cache is FP8. Head counts that are not a multiple of 16 are
|
|
# replicate-padded up to one (see _mla_fp8_prefill_attn).
|
|
from vllm.utils.torch_utils import is_quantized_kv_cache
|
|
|
|
self._fp8_prefill_enabled = _fp8_mla_prefill_supported() and (
|
|
is_quantized_kv_cache(kv_cache_dtype)
|
|
and AiterMLAHelper.is_valid_num_heads(self.num_heads)
|
|
)
|
|
if self._fp8_prefill_enabled:
|
|
from aiter import mla_prefill_ps_asm_fwd, mla_reduce_v1
|
|
|
|
self._mla_prefill_ps_asm_fwd = mla_prefill_ps_asm_fwd
|
|
self._mla_reduce_v1 = mla_reduce_v1
|
|
|
|
def _context_parallel_compute_prefill_context(
|
|
self,
|
|
q: torch.Tensor,
|
|
kv_c_and_k_pe_cache: torch.Tensor,
|
|
attn_metadata: MLACommonMetadata,
|
|
k_scale: torch.Tensor,
|
|
dcp_world_size: int,
|
|
fused_mla_kv_concat_fn: Callable[
|
|
[torch.Tensor, torch.Tensor, bool], tuple[torch.Tensor, torch.Tensor]
|
|
]
|
|
| None = None,
|
|
):
|
|
attn_metadata = cast(AiterMLAMetadata, attn_metadata)
|
|
assert attn_metadata.prefill is not None
|
|
prefill = attn_metadata.prefill
|
|
weight = getattr(self.kv_b_proj, "weight", None)
|
|
eligible = (
|
|
_supports_fused_dcp_prefill_config(
|
|
self.kv_cache_dtype,
|
|
prefill.q_data_type,
|
|
self.kv_lora_rank,
|
|
self.qk_nope_head_dim,
|
|
self.qk_rope_head_dim,
|
|
self.v_head_dim,
|
|
)
|
|
and q.dtype == torch.bfloat16
|
|
and weight is not None
|
|
and weight.dtype == torch.bfloat16
|
|
and weight.shape == (self.num_heads * 256, 512)
|
|
and weight.is_contiguous()
|
|
and getattr(self.kv_b_proj, "bias", None) is None
|
|
and getattr(self.kv_b_proj, "weight_scale", None) is None
|
|
and kv_c_and_k_pe_cache.is_contiguous()
|
|
and attn_metadata.dcp_context_row_indices is not None
|
|
)
|
|
if not eligible:
|
|
return super()._context_parallel_compute_prefill_context(
|
|
q,
|
|
kv_c_and_k_pe_cache,
|
|
attn_metadata,
|
|
k_scale,
|
|
dcp_world_size,
|
|
fused_mla_kv_concat_fn,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
|
|
assert weight is not None
|
|
assert attn_metadata.dcp_context_row_indices is not None
|
|
chunked = prefill.chunked_context
|
|
assert chunked is not None and chunked.dcp_manager is not None
|
|
assert prefill.prefill_backend is not None
|
|
assert len(chunked.chunks) == len(attn_metadata.dcp_context_row_indices)
|
|
output = output_lse = None
|
|
for chunk, row_indices in zip(
|
|
chunked.chunks, attn_metadata.dcp_context_row_indices
|
|
):
|
|
gathered = gather_compressed_context(
|
|
kv_c_and_k_pe_cache,
|
|
chunked.workspace,
|
|
prefill.block_table[chunk.request_slice],
|
|
chunk,
|
|
chunked.dcp_manager.kv_gather,
|
|
current_platform.fp8_dtype(),
|
|
)
|
|
k, v = expand_context(
|
|
gathered,
|
|
k_scale,
|
|
row_indices,
|
|
chunk.cu_seq_lens,
|
|
weight,
|
|
self.num_heads,
|
|
self.qk_nope_head_dim,
|
|
self.qk_rope_head_dim,
|
|
self.v_head_dim,
|
|
)
|
|
attn_output, attn_lse = prefill.prefill_backend.run_prefill_context_chunk(
|
|
chunk=chunk,
|
|
q=q[chunk.token_slice],
|
|
k=k,
|
|
v=v,
|
|
)
|
|
if output is None:
|
|
if len(chunked.chunks) == 1 and not chunked.empty_token_slices:
|
|
return attn_output, attn_lse
|
|
output, output_lse = init_mla_context_partial(
|
|
chunked,
|
|
attn_output,
|
|
attn_lse,
|
|
num_tokens=q.shape[0],
|
|
)
|
|
accumulate_mla_context_chunk(
|
|
chunk, attn_output, attn_lse, output, output_lse
|
|
)
|
|
return output, output_lse
|
|
|
|
def _flash_attn_varlen_diff_headdims(
|
|
self, q, k, v, return_softmax_lse=False, softmax_scale=None, **kwargs
|
|
):
|
|
output = self.flash_attn_varlen_func( # type: ignore[call-arg]
|
|
q=q,
|
|
k=k,
|
|
v=v,
|
|
softmax_scale=softmax_scale,
|
|
return_lse=return_softmax_lse,
|
|
**kwargs,
|
|
)
|
|
|
|
return output
|
|
|
|
def _mla_fp8_prefill_attn(
|
|
self,
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
v: torch.Tensor,
|
|
attn_metadata: AiterMLAMetadata,
|
|
out: torch.Tensor,
|
|
) -> None:
|
|
"""Run FP8 MLA prefill via mla_prefill_ps_asm_fwd + mla_reduce_v1.
|
|
|
|
Q, K, V are already decompressed (post-kv_b_proj), so K and V have
|
|
``num_heads`` heads (same as Q) and gqa_ratio=1. Writes the
|
|
result in-place to ``out``, which is the [total_q, nhead * v_head_dim]
|
|
output buffer supplied by ``forward_mha``; no extra allocation or
|
|
copy is required.
|
|
"""
|
|
from vllm.platforms import current_platform
|
|
from vllm.v1.worker.workspace import current_workspace_manager
|
|
|
|
fp8_dtype = current_platform.fp8_dtype()
|
|
total_q = q.shape[0]
|
|
# PS asm prefill + mla_reduce_v1 require 16-aligned heads, and the PS
|
|
# metadata is built for get_fp8_prefill_num_heads(num_heads). For head
|
|
# counts that are not a multiple of 16 (K3 = 12/rank at TP8)
|
|
# replicate-pad q/k/v up to that count, then slice the output back to
|
|
# the real head count.
|
|
#
|
|
# Counts above 16 that are not multiples of 16 (24, 40, ...) are
|
|
# handled by the same code but are not reached by any current model
|
|
# and TP that fits: 96 heads would need TP4, whose weights exceed a
|
|
# 288 GiB GPU, and 128-head models land on 128/64/32/16/8. The path is
|
|
# kept general for future architectures and its numerics are covered by
|
|
# test_fp8_prefill_matches_reference[num_heads=24]. Note the cost is
|
|
# (padded - real)/real extra FLOPs and q/k/v bytes, which is worst just
|
|
# above a multiple of 16 (17 heads pad to 32); a future arch landing
|
|
# there should measure against the flash_attn_varlen_func fallback
|
|
# rather than assume the asm path wins.
|
|
#
|
|
# Exact, not approximate: after kv_b_proj gqa_ratio is 1, so q, k and v
|
|
# all carry num_heads heads and attention is independent per head.
|
|
# Padding all three identically makes every padded head a duplicate of
|
|
# a real head, so the real heads are bit-identical to the unpadded
|
|
# result. get_mla_padded_q repeat_interleaves when the target is a
|
|
# multiple of num_heads (real heads at stride target // num_heads) and
|
|
# tiles otherwise (real heads [0:num_heads]). Same argument as the
|
|
# decode path; only the target width differs.
|
|
_real_nhead = self.num_heads
|
|
nhead = AiterMLAHelper.get_fp8_prefill_num_heads(_real_nhead)
|
|
_pad = nhead != _real_nhead
|
|
if _pad:
|
|
q = AiterMLAHelper.get_mla_padded_q(_real_nhead, q, nhead)
|
|
k = AiterMLAHelper.get_mla_padded_q(_real_nhead, k, nhead)
|
|
v = AiterMLAHelper.get_mla_padded_q(_real_nhead, v, nhead)
|
|
v_head_dim = self.v_head_dim
|
|
tile_q = _FP8_PREFILL_TILE_Q
|
|
|
|
# The FP8 ASM kernel expects FP8 inputs; the q_scale/k_scale/v_scale
|
|
# parameters select per-tensor dequant scales. Q/K/V arrive as
|
|
# bf16 from kv_b_proj, so cast here (one_scale=1.0 disables scaling).
|
|
if q.dtype != fp8_dtype:
|
|
q = q.to(fp8_dtype)
|
|
if k.dtype != fp8_dtype:
|
|
k = k.to(fp8_dtype)
|
|
if v.dtype != fp8_dtype:
|
|
v = v.to(fp8_dtype)
|
|
|
|
one_scale = torch.ones((), dtype=torch.float32, device=q.device)
|
|
|
|
# num_partial_tiles is resolved during metadata build to avoid an
|
|
# in-forward .item() sync that would prevent CUDA Graph capture.
|
|
# forward_mha gates the FP8 path on fp8_prefill_qo_indptr being set,
|
|
# and the builder always sets every fp8_prefill_* field together, so
|
|
# num_partial_tiles is non-None here.
|
|
num_partial_tiles = attn_metadata.fp8_prefill_num_partial_tiles
|
|
assert num_partial_tiles is not None
|
|
|
|
# Per-call scratch is served from the workspace manager so allocator
|
|
# churn in the prefill hot path is bounded after warmup, matching the
|
|
# pattern in PR #41002. The builder reserves the maximum shape of every
|
|
# tensor requested here before the workspace is locked.
|
|
scratch: list[tuple[tuple[int, ...], torch.dtype]] = [
|
|
((num_partial_tiles * tile_q, nhead, v_head_dim), torch.float32),
|
|
((num_partial_tiles * tile_q, nhead), torch.float32),
|
|
((total_q, nhead), torch.float32),
|
|
]
|
|
if _pad:
|
|
# The ASM and reduce kernels write a [total_q, nhead, v_head_dim]
|
|
# buffer. With unpadded heads that aliases the caller's
|
|
# [total_q, nhead * v_head_dim] output, so write straight into it;
|
|
# padded heads do not fit that storage and need their own buffer.
|
|
scratch.append(((total_q, nhead, v_head_dim), out.dtype))
|
|
|
|
workspace = current_workspace_manager()
|
|
logits, attn_lse, final_lse, *pad_out = workspace.get_simultaneous(*scratch)
|
|
out_3d = pad_out[0] if _pad else out.view(total_q, nhead, v_head_dim)
|
|
|
|
# Phase 1: persistent-scheduling assembly prefill kernel.
|
|
self._mla_prefill_ps_asm_fwd(
|
|
q,
|
|
k,
|
|
v,
|
|
attn_metadata.fp8_prefill_qo_indptr,
|
|
attn_metadata.fp8_prefill_kv_indptr,
|
|
attn_metadata.fp8_prefill_kv_indices,
|
|
attn_metadata.fp8_prefill_work_indptr,
|
|
attn_metadata.fp8_prefill_work_info_set,
|
|
attn_metadata.fp8_prefill_max_q_len,
|
|
self.scale,
|
|
True, # is_causal
|
|
logits,
|
|
attn_lse,
|
|
out_3d,
|
|
one_scale,
|
|
one_scale,
|
|
one_scale,
|
|
)
|
|
|
|
# Phase 2: reduction across KV splits.
|
|
self._mla_reduce_v1(
|
|
logits,
|
|
attn_lse,
|
|
attn_metadata.fp8_prefill_reduce_indptr,
|
|
attn_metadata.fp8_prefill_reduce_final_map,
|
|
attn_metadata.fp8_prefill_reduce_partial_map,
|
|
tile_q,
|
|
# num_kv_splits added by ROCm/aiter#3391; 0 selects the kernel
|
|
# default max(cu_num, 0) == cu_num, matching pre-#3391 behavior.
|
|
0,
|
|
out_3d,
|
|
final_lse,
|
|
)
|
|
|
|
if _pad:
|
|
real_out = (
|
|
out_3d[:, :: nhead // _real_nhead, :]
|
|
if nhead % _real_nhead == 0
|
|
else out_3d[:, :_real_nhead, :]
|
|
)
|
|
out.view(total_q, _real_nhead, v_head_dim).copy_(real_out)
|
|
|
|
def forward_mha(
|
|
self,
|
|
q: torch.Tensor,
|
|
kv_c_normed: torch.Tensor,
|
|
k_pe: torch.Tensor,
|
|
kv_c_and_k_pe_cache: torch.Tensor,
|
|
attn_metadata: MLACommonMetadata,
|
|
k_scale: torch.Tensor,
|
|
output: torch.Tensor,
|
|
output_scale: torch.Tensor | None = None,
|
|
) -> None:
|
|
"""Dispatch prefill to the FP8 ASM kernel when available.
|
|
|
|
Falls back to the parent (``flash_attn_varlen_func``) when FP8
|
|
MLA prefill is disabled, PS metadata is missing, or chunked
|
|
context requires two-pass merge.
|
|
|
|
The annotation uses the base ``MLACommonMetadata`` to honour LSP
|
|
with ``MLACommonImpl.forward_mha``; the AITER builder always
|
|
produces ``AiterMLAMetadata`` instances at runtime, so we narrow
|
|
with ``isinstance`` before reading the AITER-specific FP8 fields.
|
|
"""
|
|
if (
|
|
not self._fp8_prefill_enabled
|
|
or not isinstance(attn_metadata, AiterMLAMetadata)
|
|
or attn_metadata.fp8_prefill_qo_indptr is None
|
|
):
|
|
return super().forward_mha(
|
|
q,
|
|
kv_c_normed,
|
|
k_pe,
|
|
kv_c_and_k_pe_cache,
|
|
attn_metadata,
|
|
k_scale,
|
|
output,
|
|
output_scale,
|
|
)
|
|
|
|
assert attn_metadata.prefill is not None
|
|
prefill_metadata = attn_metadata.prefill
|
|
has_context = prefill_metadata.chunked_context is not None
|
|
|
|
if has_context:
|
|
return super().forward_mha(
|
|
q,
|
|
kv_c_normed,
|
|
k_pe,
|
|
kv_c_and_k_pe_cache,
|
|
attn_metadata,
|
|
k_scale,
|
|
output,
|
|
output_scale,
|
|
)
|
|
|
|
assert output_scale is None, (
|
|
"fused FP8 output not supported by the AITER FP8 MLA prefill path"
|
|
)
|
|
|
|
kv_nope = self.kv_b_proj(kv_c_normed)[0].view(
|
|
-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim
|
|
)
|
|
k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
|
k = self._concat_k_nope_k_pe(k_nope, k_pe)
|
|
|
|
self._mla_fp8_prefill_attn(q, k, v, attn_metadata, output)
|
|
|
|
def _forward_segmented_dcp_verify(
|
|
self,
|
|
q_nope: torch.Tensor,
|
|
q_pe: torch.Tensor,
|
|
verify: AiterMLADCPVerifyMetadata,
|
|
kv_c_and_k_pe_cache: torch.Tensor,
|
|
layer: AttentionLayer,
|
|
out_dtype: torch.dtype,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Run segmented attention over this rank's shard of every verify row.
|
|
|
|
Each row's length already covers the current tokens this rank holds, so
|
|
the cross-rank LSE merge in the MLA layer completes the causal block.
|
|
"""
|
|
q_mla = torch.cat([q_nope, q_pe], dim=-1)
|
|
# skip_reduce with NUM_SEGMENTS>1 returns the partials and never writes
|
|
# `out`. Pass None rather than aliasing q_mla: if that ever stops
|
|
# holding, this fails loudly instead of scribbling over the query.
|
|
segment_partials = _get_segmented_mla_decode()(
|
|
q_mla,
|
|
kv_c_and_k_pe_cache.view(
|
|
-1,
|
|
verify.page_size,
|
|
1,
|
|
kv_c_and_k_pe_cache.shape[-1],
|
|
),
|
|
None,
|
|
verify.qo_indptr,
|
|
verify.row_lens,
|
|
verify.max_kv_seq_len,
|
|
verify.block_table,
|
|
self.scale,
|
|
self.kv_lora_rank,
|
|
self.qk_rope_head_dim,
|
|
causal=True,
|
|
q_descale=None,
|
|
kv_descale=layer._k_scale,
|
|
skip_reduce=True,
|
|
)
|
|
assert isinstance(segment_partials, tuple) and len(segment_partials) == 3, (
|
|
"AITER segmented MLA verify must return segment partials "
|
|
"when skip_reduce=True."
|
|
)
|
|
segm_output, segm_max, segm_expsum = segment_partials
|
|
return merge_mla_segments_triton(
|
|
segm_output,
|
|
segm_max,
|
|
segm_expsum,
|
|
verify.row_lens,
|
|
verify.page_size,
|
|
out_dtype,
|
|
)
|
|
|
|
def _forward_triton_dcp_verify(
|
|
self,
|
|
q_mla: torch.Tensor,
|
|
verify: AiterMLADCPVerifyMetadata,
|
|
kv_c_and_k_pe_cache: torch.Tensor,
|
|
layer: AttentionLayer,
|
|
out_dtype: torch.dtype,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""gfx942 causal verify using the LDS-safe generic split-KV kernel.
|
|
|
|
Each verify token is its own row whose causal window is its
|
|
``row_lens`` entry. A zero-length row (an empty DCP shard or graph
|
|
padding) comes back as a NaN output with an LSE of -inf; the DCP
|
|
combine gives it zero weight and never reads the output.
|
|
"""
|
|
num_rows, num_heads = q_mla.shape[:2]
|
|
# Stage 2 stores every output and LSE element, empty rows included.
|
|
output = torch.empty(
|
|
num_rows, num_heads, self.kv_lora_rank, dtype=out_dtype, device=q_mla.device
|
|
)
|
|
# Matches merge_mla_segments_triton and the fp32 LSE the DCP combine
|
|
# kernel is warmed up for.
|
|
lse = torch.empty(num_rows, num_heads, dtype=torch.float32, device=q_mla.device)
|
|
if envs.VLLM_BATCH_INVARIANT:
|
|
num_kv_splits = 1
|
|
else:
|
|
num_kv_splits = _triton_dcp_verify_num_kv_splits(
|
|
verify.max_kv_seq_len, num_rows, num_heads, self._sm_count
|
|
)
|
|
logits_shape = (num_rows, num_heads, num_kv_splits, self.kv_lora_rank + 1)
|
|
if is_workspace_manager_initialized():
|
|
(attn_logits,) = current_workspace_manager().get_simultaneous(
|
|
(logits_shape, torch.float32),
|
|
)
|
|
else:
|
|
attn_logits = torch.empty(
|
|
logits_shape, dtype=torch.float32, device=q_mla.device
|
|
)
|
|
|
|
paged_kv = kv_c_and_k_pe_cache.unsqueeze(2)
|
|
decode_attention_fwd(
|
|
q_mla,
|
|
paged_kv,
|
|
paged_kv[..., : self.kv_lora_rank],
|
|
output,
|
|
lse,
|
|
verify.block_table,
|
|
verify.row_lens,
|
|
attn_logits,
|
|
num_kv_splits,
|
|
self.scale,
|
|
paged_kv.size(1),
|
|
k_scale=layer._k_scale,
|
|
v_scale=layer._k_scale,
|
|
is_mla=True,
|
|
)
|
|
return output, lse
|
|
|
|
def forward_mqa(
|
|
self,
|
|
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
|
kv_c_and_k_pe_cache: torch.Tensor,
|
|
attn_metadata: AiterMLAMetadata,
|
|
layer: AttentionLayer,
|
|
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
|
decode = attn_metadata.decode
|
|
if decode is None or decode.varlen_q_rows is None:
|
|
return self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
|
|
assert decode.varlen_o_rows is not None
|
|
q_rows = decode.varlen_q_rows
|
|
if isinstance(q, tuple):
|
|
num_tokens = q[0].shape[0]
|
|
q = (q[0].index_select(0, q_rows), q[1].index_select(0, q_rows))
|
|
else:
|
|
num_tokens = q.shape[0]
|
|
q = q.index_select(0, q_rows)
|
|
o, lse = self._forward_mqa(q, kv_c_and_k_pe_cache, attn_metadata, layer)
|
|
assert lse is None
|
|
return o.index_select(0, decode.varlen_o_rows[:num_tokens]), None
|
|
|
|
def _forward_mqa(
|
|
self,
|
|
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
|
kv_c_and_k_pe_cache: torch.Tensor,
|
|
attn_metadata: AiterMLAMetadata,
|
|
layer: AttentionLayer,
|
|
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
|
assert kv_c_and_k_pe_cache.numel() > 0
|
|
assert attn_metadata.decode is not None
|
|
|
|
decode = attn_metadata.decode
|
|
assert decode.max_qo_len is not None
|
|
if decode.use_gluon_decode:
|
|
assert decode.paged_kv_indptr is not None
|
|
assert decode.paged_kv_indices is not None
|
|
if type(q) is tuple:
|
|
q_nope, q_pe = q
|
|
else:
|
|
q_nope, q_pe = torch.split(
|
|
q, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
|
)
|
|
B, num_q_heads, _ = q_nope.shape
|
|
o = torch.empty(
|
|
B,
|
|
num_q_heads,
|
|
self.kv_lora_rank,
|
|
dtype=decode.attn_out_dtype,
|
|
device=q_nope.device,
|
|
)
|
|
kv_buffer = kv_c_and_k_pe_cache.reshape(-1, kv_c_and_k_pe_cache.shape[-1])
|
|
mla_gluon = _get_mla_gluon()
|
|
mla_gluon(
|
|
q_nope=q_nope,
|
|
q_pe=q_pe,
|
|
kv_c=kv_buffer,
|
|
o=o,
|
|
page_table=decode.paged_kv_indices,
|
|
seq_info=decode.paged_kv_indptr,
|
|
sm_scale=self.scale,
|
|
k_pe=None,
|
|
kv_pe_offset=self.kv_lora_rank,
|
|
use_2d_view=False,
|
|
kv_scale=1.0,
|
|
min_kv_seq_len=decode.min_kv_seq_len,
|
|
)
|
|
return o, None
|
|
|
|
# 12-head (<16) multi-token verify (DSpark): the asm path has no
|
|
# gqa<16, qseqlen>1 kernel, so the block goes to the gluon kernel's 4-D
|
|
# MTP entry, which serves a whole (1 + num_spec) block in one launch.
|
|
# The block is causal -- the target is checking draft tokens, so
|
|
# position t must not see t+1 -- and the kernel bounds each query
|
|
# position's scores itself.
|
|
# Arch, mode and dtype gating all live in use_gluon_verify, which the
|
|
# builder evaluates and records on the metadata, so this branch acts on
|
|
# the same answer the builder used to decide whether the asm decode
|
|
# would run. DCP verify is routed to the segmented path below instead.
|
|
if decode.use_gluon_verify:
|
|
if type(q) is tuple:
|
|
q_nope, q_pe = q
|
|
else:
|
|
q_nope, q_pe = torch.split(
|
|
q, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
|
)
|
|
B, num_q_heads, _ = q_nope.shape
|
|
o = torch.empty(
|
|
B,
|
|
num_q_heads,
|
|
self.kv_lora_rank,
|
|
dtype=decode.attn_out_dtype,
|
|
device=q_nope.device,
|
|
)
|
|
kv_buffer = kv_c_and_k_pe_cache.reshape(-1, kv_c_and_k_pe_cache.shape[-1])
|
|
assert attn_metadata.causal, (
|
|
"AITER MLA small-head verify MTP is causal-only"
|
|
)
|
|
# Hand mla_gluon its 4-D MTP entry instead of an expanded
|
|
# per-verify-token paged-KV view. The flat query layout is already
|
|
# row-major (r * qlen + t), so unflatten is a free view. mla_gluon
|
|
# applies the per-position causal bound
|
|
# score_end = min(split_kv_end, seq_len - qlen + q_pos + 1)
|
|
# in-kernel, and seq_lens already spans the verify block, so that
|
|
# bound is context_r + q_pos + 1 -- the same window the expanded
|
|
# view supplied by truncating each row's page list.
|
|
qlen = int(decode.max_qo_len)
|
|
num_reqs = B // qlen
|
|
if num_reqs * qlen != B:
|
|
raise ValueError(
|
|
f"verify block {B} rows is not a multiple of qlen {qlen}"
|
|
)
|
|
assert decode.paged_kv_indptr is not None
|
|
assert decode.paged_kv_indices is not None
|
|
mla_gluon = _get_mla_gluon()
|
|
mla_gluon(
|
|
q_nope=q_nope.unflatten(0, (num_reqs, qlen)),
|
|
q_pe=q_pe.unflatten(0, (num_reqs, qlen)),
|
|
kv_c=kv_buffer,
|
|
o=o.unflatten(0, (num_reqs, qlen)),
|
|
page_table=decode.paged_kv_indices,
|
|
seq_info=decode.paged_kv_indptr,
|
|
sm_scale=self.scale,
|
|
k_pe=None,
|
|
kv_pe_offset=self.kv_lora_rank,
|
|
use_2d_view=False,
|
|
kv_scale=1.0,
|
|
min_kv_seq_len=decode.min_kv_seq_len,
|
|
)
|
|
return o, None
|
|
|
|
dcp_route = decode.dcp_route
|
|
if dcp_route is _DCPDecodeRoute.TRITON:
|
|
verify = decode.dcp_verify
|
|
assert verify is not None
|
|
q_mla = torch.cat(q, dim=-1) if isinstance(q, tuple) else q
|
|
if (
|
|
is_quantized_kv_cache(self.kv_cache_dtype)
|
|
and q_mla.dtype != torch.bfloat16
|
|
):
|
|
q_mla = q_mla.to(torch.bfloat16) * layer._q_scale
|
|
return self._forward_triton_dcp_verify(
|
|
q_mla,
|
|
verify,
|
|
kv_c_and_k_pe_cache,
|
|
layer,
|
|
decode.attn_out_dtype,
|
|
)
|
|
|
|
if dcp_route is _DCPDecodeRoute.SEGMENTED:
|
|
verify = decode.dcp_verify
|
|
assert verify is not None
|
|
if type(q) is tuple:
|
|
q_nope, q_pe = q
|
|
else:
|
|
q_nope, q_pe = torch.split(
|
|
q, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
|
)
|
|
if (
|
|
is_quantized_kv_cache(self.kv_cache_dtype)
|
|
and q_nope.dtype != torch.bfloat16
|
|
):
|
|
q_nope = q_nope.to(torch.bfloat16) * layer._q_scale
|
|
q_pe = q_pe.to(torch.bfloat16) * layer._q_scale
|
|
return self._forward_segmented_dcp_verify(
|
|
q_nope,
|
|
q_pe,
|
|
verify,
|
|
kv_c_and_k_pe_cache,
|
|
layer,
|
|
decode.attn_out_dtype,
|
|
)
|
|
|
|
if type(q) is tuple:
|
|
q = torch.cat(q, dim=-1)
|
|
|
|
assert isinstance(q, torch.Tensor)
|
|
assert decode.paged_kv_indptr is not None
|
|
assert decode.paged_kv_indices is not None
|
|
B = q.shape[0]
|
|
|
|
assert q.shape[1] == self._decode_num_heads, (
|
|
"ROCM_AITER_MLA decode expected the DCP-gathered query head count "
|
|
f"{self._decode_num_heads}, got {q.shape[1]}"
|
|
)
|
|
# cprr has kernels only at native head counts, so pad q up (e.g. 96 ->
|
|
# 128). Heads are independent in MLA; the extras are sliced off below.
|
|
asm_dcp_heads = decode.asm_decode_num_heads
|
|
if asm_dcp_heads > self._decode_num_heads:
|
|
mla_num_heads = asm_dcp_heads
|
|
mla_padded_q = AiterMLAHelper.get_mla_padded_q(
|
|
self._decode_num_heads, q, target_heads=mla_num_heads
|
|
)
|
|
else:
|
|
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
|
|
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(
|
|
self._decode_num_heads
|
|
)
|
|
o = torch.empty(
|
|
B,
|
|
mla_num_heads,
|
|
self.kv_lora_rank,
|
|
dtype=attn_metadata.decode.attn_out_dtype,
|
|
device=q.device,
|
|
)
|
|
if decode.max_qo_len > 1 and not decode.has_persistent_metadata:
|
|
# MTP verification can call the AITER MLA decode kernel with
|
|
# qlen > 1. If that path is running without persistent metadata,
|
|
# zero-fill so unwritten lanes cannot leak into logits.
|
|
o.zero_()
|
|
|
|
kv_buffer = kv_c_and_k_pe_cache.unsqueeze(2)
|
|
|
|
# Build kwargs for mla_decode_fwd. Pass persistent metadata only
|
|
# when it was successfully computed.
|
|
mla_kwargs = dict(
|
|
q_scale=layer._q_scale,
|
|
kv_scale=layer._k_scale,
|
|
)
|
|
if attn_metadata.work_meta_data is not None:
|
|
mla_kwargs.update(
|
|
work_meta_data=attn_metadata.work_meta_data,
|
|
work_indptr=attn_metadata.work_indptr,
|
|
work_info_set=attn_metadata.work_info_set,
|
|
reduce_indptr=attn_metadata.reduce_indptr,
|
|
reduce_final_map=attn_metadata.reduce_final_map,
|
|
reduce_partial_map=attn_metadata.reduce_partial_map,
|
|
)
|
|
|
|
lse = None
|
|
if self.dcp_world_size > 1:
|
|
if dcp_route is _DCPDecodeRoute.CPRR:
|
|
# cprr only: global positions for the in-kernel causal window.
|
|
assert decode.g_kv_indptr is not None
|
|
mla_kwargs["g_kv_indptr"] = decode.g_kv_indptr
|
|
mla_kwargs["cp_world_size"] = decode.cp_world_size
|
|
mla_kwargs["cp_rank"] = decode.cp_rank
|
|
# Must match the schedule get_mla_metadata_v1 was built with.
|
|
assert decode.mla_num_kv_splits > 0
|
|
mla_kwargs["num_kv_splits"] = decode.mla_num_kv_splits
|
|
mla_kwargs["intra_batch_mode"] = False
|
|
# The vLLM custom-op wrapper exposes only the in-place output and
|
|
# drops aiter's final LSE, which the cross-shard merge needs, so go
|
|
# through aiter's native entry point on the DCP path.
|
|
_, lse = _get_aiter_mla_decode()(
|
|
mla_padded_q,
|
|
kv_buffer.view(-1, 1, 1, mla_padded_q.shape[-1]),
|
|
o,
|
|
decode.qo_indptr,
|
|
decode.paged_kv_indptr,
|
|
decode.paged_kv_indices,
|
|
decode.paged_kv_last_page_len,
|
|
decode.max_qo_len,
|
|
sm_scale=self.scale,
|
|
return_lse=True,
|
|
causal=attn_metadata.causal,
|
|
**mla_kwargs,
|
|
)
|
|
assert lse is not None, (
|
|
"aiter mla_decode_fwd(return_lse=True) returned no LSE; upgrade "
|
|
"aiter to a build with decode LSE support."
|
|
)
|
|
else:
|
|
rocm_aiter_ops.mla_decode_fwd(
|
|
mla_padded_q,
|
|
kv_buffer,
|
|
o,
|
|
self.scale,
|
|
decode.qo_indptr,
|
|
decode.max_qo_len,
|
|
decode.paged_kv_indptr,
|
|
decode.paged_kv_indices,
|
|
decode.paged_kv_last_page_len,
|
|
causal=attn_metadata.causal,
|
|
**mla_kwargs,
|
|
)
|
|
|
|
if asm_dcp_heads > self._decode_num_heads:
|
|
# Keep a view: a post-kernel copy races the symm-mem a2a combine.
|
|
output = o[:, : self._decode_num_heads]
|
|
if lse is not None:
|
|
lse = lse[:, : self._decode_num_heads, ...]
|
|
else:
|
|
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
|
|
if lse is not None:
|
|
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
|
|
return output, lse
|