[ROCm][DSv4.1][Perf] Emit MXFP8 from the sparse decode reduce and run wo_a as a grouped FP8 GEMM (#58456)

Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Fangzhou Ai
2026-09-24 11:58:14 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent dcfc17e0b1
commit 721d0e5c11
3 changed files with 664 additions and 29 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from types import SimpleNamespace from types import SimpleNamespace
import pytest import pytest
@@ -261,14 +262,19 @@ def _launch_sparse_decode_reduce(
adaptive_splits: bool, adaptive_splits: bool,
positions: torch.Tensor | None = None, positions: torch.Tensor | None = None,
cos_sin_cache: torch.Tensor | None = None, cos_sin_cache: torch.Tensor | None = None,
out_dtype: torch.dtype = torch.bfloat16,
out_scale: torch.Tensor | None = None,
) -> torch.Tensor: ) -> torch.Tensor:
"""Run the reduce kernel, with the inverse-RoPE epilogue if given positions.""" """Run the reduce kernel, with the inverse-RoPE epilogue if given positions.
Passing ``out_scale`` selects the MXFP8 epilogue (``out_dtype`` e4m3).
"""
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
num_queries, num_splits, num_heads = part_m.shape num_queries, num_splits, num_heads = part_m.shape
out = torch.empty( out = torch.empty(
(num_queries, num_heads, HEAD_DIM), (num_queries, num_heads, HEAD_DIM),
dtype=torch.bfloat16, dtype=out_dtype,
device=part_m.device, device=part_m.device,
) )
attn_sink = torch.empty(1, dtype=torch.float32, device=part_m.device) attn_sink = torch.empty(1, dtype=torch.float32, device=part_m.device)
@@ -280,8 +286,11 @@ def _launch_sparse_decode_reduce(
out, out,
positions, positions,
cos_sin_cache, cos_sin_cache,
out_scale,
out.stride(0), out.stride(0),
out.stride(1), out.stride(1),
out_scale.stride(0) if out_scale is not None else 0,
HEAD_DIM // 32,
part_m.stride(0), part_m.stride(0),
part_m.stride(1), part_m.stride(1),
part_acc.stride(0), part_acc.stride(0),
@@ -298,6 +307,7 @@ def _launch_sparse_decode_reduce(
FUSE_INV_ROPE=positions is not None, FUSE_INV_ROPE=positions is not None,
NOPE=NOPE_HEAD_DIM, NOPE=NOPE_HEAD_DIM,
HALF=ROPE_HEAD_DIM // 2, HALF=ROPE_HEAD_DIM // 2,
QUANT_OUT=out_scale is not None,
num_warps=4, num_warps=4,
) )
return out return out
@@ -1057,6 +1067,171 @@ def test_sparse_attn_decode_reduce_inverse_rope_epilogue(adaptive_splits: bool)
assert not torch.equal(actual[..., NOPE_HEAD_DIM:], unfused[..., NOPE_HEAD_DIM:]) assert not torch.equal(actual[..., NOPE_HEAD_DIM:], unfused[..., NOPE_HEAD_DIM:])
def _mxfp8_dequant(data: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
blocks = data.float().view(*data.shape[:-1], -1, 32)
return (blocks * torch.exp2(scale.float() - 127.0)[..., None]).view(data.shape)
def _random_cos_sin_cache(max_pos: int, device: torch.device) -> torch.Tensor:
angle = torch.randn(max_pos, ROPE_HEAD_DIM // 2, device=device)
return torch.cat((angle.cos(), angle.sin()), dim=-1).contiguous()
@requires_gfx950
@torch.inference_mode()
def test_sparse_attn_decode_reduce_mxfp8_epilogue() -> None:
"""QUANT_OUT must equal MXFP8-quantizing the reduce's fp32 output."""
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
_mxfp8_e4m3_quantize_torch,
)
device = torch.device("cuda")
torch.manual_seed(7)
num_queries, num_splits, num_heads = 5, 4, 8
shape = (num_queries, num_splits, num_heads)
part_m = torch.randn(shape, device=device)
part_l = torch.rand(shape, device=device) + 0.5
part_acc = torch.randn((*shape, HEAD_DIM), device=device)
positions = torch.randint(0, 64, (num_queries,), device=device)
cos_sin_cache = _random_cos_sin_cache(64, device)
args = (part_m, part_l, part_acc, False, positions, cos_sin_cache)
rotated = _launch_sparse_decode_reduce(*args, out_dtype=torch.float32)
expected_data, expected_scale = _mxfp8_e4m3_quantize_torch(
rotated.view(num_queries, -1)
)
scale = torch.empty_like(expected_scale)
data = _launch_sparse_decode_reduce(
*args, out_dtype=torch.float8_e4m3fn, out_scale=scale
)
assert torch.equal(scale, expected_scale)
assert torch.equal(
data.view(num_queries, -1).view(torch.uint8), expected_data.view(torch.uint8)
)
@requires_gfx950
@torch.inference_mode()
def test_inverse_rope_mxfp8_rows() -> None:
"""Prefill rows get the same inverse RoPE + MXFP8 as the decode epilogue."""
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
_mxfp8_e4m3_quantize_torch,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_inverse_rope_mxfp8_rows,
)
device = torch.device("cuda")
torch.manual_seed(5)
num_tokens, num_heads = 7, 4
o = torch.randn(num_tokens, num_heads, HEAD_DIM, device=device).bfloat16()
positions = torch.randint(0, 64, (num_tokens,), device=device)
cos_sin_cache = _random_cos_sin_cache(64, device)
ref = o.float()
cos, sin = cos_sin_cache[positions, None].chunk(2, dim=-1)
even, odd = ref[..., NOPE_HEAD_DIM::2].clone(), ref[..., NOPE_HEAD_DIM + 1 :: 2]
ref[..., NOPE_HEAD_DIM::2] = even * cos + odd * sin
ref[..., NOPE_HEAD_DIM + 1 :: 2] = odd * cos - even * sin
ref = ref.view(num_tokens, -1)
_, expected_scale = _mxfp8_e4m3_quantize_torch(ref)
data = torch.empty_like(ref, dtype=torch.float8_e4m3fn)
scale = torch.empty_like(expected_scale)
rocm_inverse_rope_mxfp8_rows(
o, positions, cos_sin_cache, ROPE_HEAD_DIM, data, scale
)
assert torch.equal(scale, expected_scale)
# Within half an e4m3 step at the top of each block's range.
half_step = 16.0 * torch.exp2(scale.float() - 127.0).repeat_interleave(32, -1)
assert bool(((_mxfp8_dequant(data, scale) - ref).abs() <= half_step).all())
@requires_gfx950
@torch.inference_mode()
def test_sparse_attn_decode_mxfp8_output() -> None:
"""``out_mxfp8`` routes the real decode through the MXFP8 epilogue."""
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_decode_ragged_triton,
)
device = torch.device("cuda")
torch.manual_seed(1)
num_heads = 16
q = torch.randn(2, num_heads, HEAD_DIM, device=device).bfloat16() * 0.125
kv = torch.randn(6, HEAD_DIM, device=device).bfloat16() * 0.125
decode = functools.partial(
_rocm_sparse_attn_decode_ragged_triton,
q=q,
main_cache=_pack_fp8_ds_mla_cache(kv, 4, current_platform.is_fp8_fnuz()),
main_indices=torch.tensor([0, 2, 4, 1], dtype=torch.int32, device=device),
main_indptr=torch.tensor([0, 2, 4], dtype=torch.int32, device=device),
scale=HEAD_DIM**-0.5,
attn_sink=None,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
inv_rope_positions=torch.tensor([3, 9], device=device),
inv_rope_cos_sin_cache=_random_cos_sin_cache(16, device),
)
expected = decode().float().view(2, -1)
data = torch.empty(
2, num_heads * HEAD_DIM, dtype=torch.float8_e4m3fn, device=device
)
scale = torch.empty(2, num_heads * HEAD_DIM // 32, dtype=torch.uint8, device=device)
decode(out_mxfp8=(data, scale))
torch.testing.assert_close(
_mxfp8_dequant(data, scale), expected, atol=1e-3, rtol=2**-4
)
@requires_gfx950
@pytest.mark.parametrize(
"num_tokens, n_groups",
# One case per tile tier of _mxfp8_wo_a_bmm_config, with partial M tiles.
[
(1, 4),
(20, 4),
(48, 4),
(77, 4),
(130, 4),
(300, 4),
(700, 4),
(1000, 4),
(1100, 8),
],
)
@torch.inference_mode()
def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None:
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
_mxfp8_e4m3_quantize_torch,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import rocm_mxfp8_wo_a_bmm
device = torch.device("cuda")
torch.manual_seed(3)
o_lora_rank, group_dim = 128, 2048
a, a_scale = _mxfp8_e4m3_quantize_torch(
torch.randn(num_tokens, n_groups * group_dim, device=device)
)
w, w_scale = _mxfp8_e4m3_quantize_torch(
torch.randn(n_groups * o_lora_rank, group_dim, device=device)
)
wo_a = SimpleNamespace(weight=w, weight_scale=w_scale)
out = rocm_mxfp8_wo_a_bmm(a, a_scale, wo_a, n_groups, o_lora_rank)
expected = torch.einsum(
"tgd,grd->tgr",
_mxfp8_dequant(a, a_scale).view(num_tokens, n_groups, group_dim),
_mxfp8_dequant(w, w_scale).view(n_groups, o_lora_rank, group_dim),
)
torch.testing.assert_close(out.float(), expected.flatten(1), atol=5e-2, rtol=1e-2)
@requires_gfx950 @requires_gfx950
@pytest.mark.parametrize("extra_len", [0, 1, 31, 32, 33, 63, 64, 65]) @pytest.mark.parametrize("extra_len", [0, 1, 31, 32, 33, 63, 64, 65])
@torch.inference_mode() @torch.inference_mode()
+136 -22
View File
@@ -13,6 +13,8 @@ from vllm.distributed import (
) )
from vllm.forward_context import get_forward_context from vllm.forward_context import get_forward_context
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
from vllm.models.deepseek_v41.attention import ( from vllm.models.deepseek_v41.attention import (
DeepseekV4Attention, DeepseekV4Attention,
_replace_layer_index, _replace_layer_index,
@@ -37,7 +39,9 @@ from vllm.v1.attention.backends.mla.sparse_swa import (
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
build_ragged_indices_from_dense, build_ragged_indices_from_dense,
rocm_inv_rope_einsum, rocm_inv_rope_einsum,
rocm_inverse_rope_mxfp8_rows,
rocm_inverse_rope_rows_, rocm_inverse_rope_rows_,
rocm_mxfp8_wo_a_bmm,
rocm_sparse_attn_decode, rocm_sparse_attn_decode,
rocm_sparse_attn_prefill, rocm_sparse_attn_prefill,
) )
@@ -663,7 +667,40 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
transpose_scale=False, transpose_scale=False,
) )
def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: def _alloc_attn_out(
self, num_tokens: int, hidden_states: torch.Tensor
) -> torch.Tensor | QuantizedActivation:
if not _ON_GFX950:
return super()._alloc_attn_out(num_tokens, hidden_states)
# wo_a's MXFP8 input: the decode reduce writes it directly, prefill
# rows are rotated and quantized after their bf16 attention.
width = self.n_local_heads * self.head_dim
data = torch.empty(
(num_tokens, width), dtype=torch.float8_e4m3fn, device=hidden_states.device
)
scale = torch.empty(
(num_tokens, width // 32), dtype=torch.uint8, device=hidden_states.device
)
return QuantizedActivation(
data=data,
scale=scale,
orig_dtype=hidden_states.dtype,
orig_shape=data.shape,
quant_key=kMxfp8Dynamic,
)
def _o_proj(
self, attn_out: torch.Tensor | QuantizedActivation, positions: torch.Tensor
) -> torch.Tensor:
if isinstance(attn_out, QuantizedActivation):
z = rocm_mxfp8_wo_a_bmm(
attn_out.data,
attn_out.scale,
self.wo_a,
self.n_local_groups,
self.o_lora_rank,
)
return self._wo_b_after_wo_a(z)
o = attn_out[:, : self.n_local_heads, :] o = attn_out[:, : self.n_local_heads, :]
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b. # ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
z = rocm_inv_rope_einsum( z = rocm_inv_rope_einsum(
@@ -676,26 +713,29 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
self.wo_a, self.wo_a,
inverse_rope=False, inverse_rope=False,
) )
zf = z.flatten(1) return self._wo_b_after_wo_a(z.flatten(1))
def _wo_b_after_wo_a(self, zf: torch.Tensor) -> torch.Tensor:
if self._wo_b_scale is not None and zf.dim() == 2: if self._wo_b_scale is not None and zf.dim() == 2:
result = self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True) return self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True)
else: return self.wo_b(zf)
result = self.wo_b(zf)
return result
def forward_mqa( def forward_mqa(
self, self,
q: torch.Tensor, q: torch.Tensor,
kv: torch.Tensor, kv: torch.Tensor,
positions: torch.Tensor, positions: torch.Tensor,
output: torch.Tensor, output: torch.Tensor | QuantizedActivation,
) -> None: ) -> None:
assert output.shape == q.shape, ( mxfp8_out = output if isinstance(output, QuantizedActivation) else None
f"output buffer shape {output.shape} must match q shape {q.shape}" if mxfp8_out is None:
) assert isinstance(output, torch.Tensor)
assert output.dtype == q.dtype, ( assert output.shape == q.shape, (
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}" f"output buffer shape {output.shape} must match q shape {q.shape}"
) )
assert output.dtype == q.dtype, (
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
)
forward_context = get_forward_context() forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata attn_metadata = forward_context.attn_metadata
@@ -713,9 +753,13 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
) )
M = N + self.window_size + self.max_num_batched_tokens M = N + self.window_size + self.max_num_batched_tokens
current_workspace_manager().get_simultaneous( current_workspace_manager().get_simultaneous(
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), *self._prefill_workspace_shapes(M, self.max_num_batched_tokens, q)
) )
output.zero_() if mxfp8_out is None:
output.zero_()
else:
mxfp8_out.data.zero_()
mxfp8_out.scale.zero_()
return return
assert isinstance(attn_metadata, dict) assert isinstance(attn_metadata, dict)
@@ -745,10 +789,41 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
positions=positions[num_decode_tokens:], positions=positions[num_decode_tokens:],
compressed_k_cache=self_kv_cache, compressed_k_cache=self_kv_cache,
swa_k_cache=swa_kv_cache, swa_k_cache=swa_kv_cache,
output=output[num_decode_tokens:], output=(
output[num_decode_tokens:]
if isinstance(output, torch.Tensor)
else None
),
attn_metadata=rocm_metadata, attn_metadata=rocm_metadata,
swa_metadata=swa_metadata, swa_metadata=swa_metadata,
mxfp8_out=(
None
if mxfp8_out is None
else (
mxfp8_out.data[num_decode_tokens:],
mxfp8_out.scale[num_decode_tokens:],
)
),
) )
if mxfp8_out is not None:
# The reduce epilogue rotates and quantizes every decode row and
# the prefill path already did its own, so nothing is owed here.
if num_decodes > 0:
self._forward_decode(
q=q[:num_decode_tokens],
positions=positions[:num_decode_tokens],
kv_cache=self_kv_cache,
swa_metadata=swa_metadata,
attn_metadata=rocm_metadata,
swa_only=swa_only,
output=None,
output_mxfp8=(
mxfp8_out.data[:num_decode_tokens],
mxfp8_out.scale[:num_decode_tokens],
),
)
return
assert isinstance(output, torch.Tensor)
rotated = 0 rotated = 0
if num_decodes > 0: if num_decodes > 0:
rotated = self._forward_decode( rotated = self._forward_decode(
@@ -816,7 +891,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
attn_metadata: DeepseekV4FlashMLAMetadata | None, attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_only: bool, swa_only: bool,
output: torch.Tensor, output: torch.Tensor | None,
output_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> int: ) -> int:
"""Returns how many leading rows the decode epilogue inverse-RoPE'd.""" """Returns how many leading rows the decode epilogue inverse-RoPE'd."""
num_decodes = swa_metadata.num_decodes num_decodes = swa_metadata.num_decodes
@@ -856,6 +932,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
nope_head_dim=self.nope_head_dim, nope_head_dim=self.nope_head_dim,
rope_head_dim=self.rope_head_dim, rope_head_dim=self.rope_head_dim,
output=output, output=output,
output_mxfp8=output_mxfp8,
inv_rope_positions=positions, inv_rope_positions=positions,
inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache, inv_rope_cos_sin_cache=self.rotary_emb.cos_sin_cache,
extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free( extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free(
@@ -871,10 +948,16 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
positions: torch.Tensor, positions: torch.Tensor,
compressed_k_cache: torch.Tensor | None, compressed_k_cache: torch.Tensor | None,
swa_k_cache: torch.Tensor, swa_k_cache: torch.Tensor,
output: torch.Tensor, output: torch.Tensor | None,
attn_metadata: DeepseekV4FlashMLAMetadata | None, attn_metadata: DeepseekV4FlashMLAMetadata | None,
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
mxfp8_out: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> None: ) -> None:
"""Sparse prefill into bf16 ``output``, or into ``mxfp8_out``.
With ``mxfp8_out`` the rows go to a bf16 workspace first and are then
inverse-RoPE'd and quantized into it in one pass.
"""
swa_only = attn_metadata is None swa_only = attn_metadata is None
num_prefills = swa_metadata.num_prefills num_prefills = swa_metadata.num_prefills
@@ -910,10 +993,13 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
self.PREFILL_CHUNK_SIZE self.PREFILL_CHUNK_SIZE
) )
workspace_manager = current_workspace_manager() workspace = current_workspace_manager().get_simultaneous(
kv = workspace_manager.get_simultaneous( *self._prefill_workspace_shapes(M, num_prefill_tokens, q)
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), )
)[0] kv = workspace[0]
if mxfp8_out is not None:
output = workspace[1]
assert output is not None
for chunk_idx in range(num_chunks): for chunk_idx in range(num_chunks):
chunk_start = chunk_idx * self.PREFILL_CHUNK_SIZE chunk_start = chunk_idx * self.PREFILL_CHUNK_SIZE
chunk_end = min(chunk_start + self.PREFILL_CHUNK_SIZE, num_prefills) chunk_end = min(chunk_start + self.PREFILL_CHUNK_SIZE, num_prefills)
@@ -978,3 +1064,31 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
attn_sink=self.attn_sink, attn_sink=self.attn_sink,
output=output[query_start:query_end], output=output[query_start:query_end],
) )
if mxfp8_out is not None:
rocm_inverse_rope_mxfp8_rows(
output,
positions[:num_prefill_tokens],
self.rotary_emb.cos_sin_cache,
self.rope_head_dim,
mxfp8_out[0][:num_prefill_tokens],
mxfp8_out[1][:num_prefill_tokens],
)
def _prefill_workspace_shapes(
self, M: int, num_prefill_tokens: int, q: torch.Tensor
) -> list[tuple[tuple[int, ...], torch.dtype]]:
"""The prefill gather buffer, plus bf16 rows for the MXFP8 output.
One ``get_simultaneous`` call: separate calls alias the same memory.
"""
shapes: list[tuple[tuple[int, ...], torch.dtype]] = [
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16)
]
if _ON_GFX950:
shapes.append(
(
(num_prefill_tokens, self.n_local_heads, self.head_dim),
torch.bfloat16,
)
)
return shapes
+351 -5
View File
@@ -16,7 +16,7 @@ from vllm.forward_context import get_forward_context
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import LayerNameType from vllm.utils.torch_utils import LayerNameType, direct_register_custom_op
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
from vllm.v1.worker.workspace import current_workspace_manager from vllm.v1.worker.workspace import current_workspace_manager
@@ -1637,6 +1637,90 @@ def rocm_inverse_rope_rows_(
_fused_inverse_rope_gptj(o, positions, cos_sin_cache, rope_head_dim, out=o) _fused_inverse_rope_gptj(o, positions, cos_sin_cache, rope_head_dim, out=o)
@triton.jit
def _inverse_rope_mxfp8_quant_kernel(
o_ptr, # [T, H, D] bf16
q_ptr, # [T, H * D] e4m3
s_ptr, # [T, H * D // 32] uint8 (E8M0)
pos_ptr,
cos_sin_ptr,
s_t,
s_h,
qs_t,
ss_t,
cs_stride,
HEAD_DIM: tl.constexpr,
NOPE: tl.constexpr,
HALF: tl.constexpr,
):
t = tl.program_id(0)
h = tl.program_id(1)
lanes = tl.arange(0, HEAD_DIM)
x = tl.load(o_ptr + t * s_t + h * s_h + lanes).to(tl.float32)
# Same lane-pair formulation as the reduce epilogue: NoPE pairs take
# cos=1 / sin=0, so one expression rotates the whole row.
pos = tl.load(pos_ptr + t).to(tl.int64)
pair_idx = tl.arange(0, HEAD_DIM // 2) - (NOPE // 2)
is_rope = pair_idx >= 0
k = tl.where(is_rope, pair_idx, 0)
cos = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + k), 1.0)
sin = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + HALF + k), 0.0)
even, odd = tl.split(tl.reshape(x, (HEAD_DIM // 2, 2)))
x = tl.reshape(
tl.join(even * cos + odd * sin, odd * cos - even * sin), (1, HEAD_DIM)
)
xq, bits = _mxfp8_quantize_rows(x, 1, HEAD_DIM)
tl.store(
q_ptr + t * qs_t + h * HEAD_DIM + lanes,
tl.reshape(xq, (HEAD_DIM,)).to(q_ptr.dtype.element_ty),
)
scale_offsets = tl.arange(0, HEAD_DIM // 32)
tl.store(
s_ptr + t * ss_t + h * (HEAD_DIM // 32) + scale_offsets,
tl.reshape(bits, (HEAD_DIM // 32,)).to(tl.uint8),
)
def rocm_inverse_rope_mxfp8_rows(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
rope_head_dim: int,
out_data: torch.Tensor,
out_scale: torch.Tensor,
) -> None:
"""Inverse-RoPE bf16 attention rows and MXFP8-quantize them for wo_a.
The counterpart of ``rocm_inverse_rope_rows_`` for layers whose attention
output is MXFP8: rows the decode reduce did not emit (prefill) go through
here. ``o`` is [T, H, D]; ``out_data`` [T, H * D] e4m3 and ``out_scale``
[T, H * D // 32] E8M0, the layout the reduce epilogue writes.
"""
num_tokens, num_heads, head_dim = o.shape
if num_tokens == 0:
return
assert o.stride(-1) == 1 and out_data.stride(-1) == 1 and out_scale.stride(-1) == 1
assert out_data.shape == (num_tokens, num_heads * head_dim)
assert out_scale.shape == (num_tokens, num_heads * head_dim // 32)
assert cos_sin_cache.shape[-1] == rope_head_dim
_inverse_rope_mxfp8_quant_kernel[(num_tokens, num_heads)](
o,
out_data,
out_scale,
positions,
cos_sin_cache,
o.stride(0),
o.stride(1),
out_data.stride(0),
out_scale.stride(0),
cos_sin_cache.stride(0),
HEAD_DIM=head_dim,
NOPE=head_dim - rope_head_dim,
HALF=rope_head_dim // 2,
num_warps=4,
)
def _get_cached_wo_a_bf16( def _get_cached_wo_a_bf16(
wo_a: torch.nn.Module, wo_a: torch.nn.Module,
n_local_groups: int, n_local_groups: int,
@@ -1722,6 +1806,175 @@ def rocm_inv_rope_einsum(
return torch.einsum("tgd,grd->tgr", o_ref, wo_a_weight) return torch.einsum("tgd,grd->tgr", o_ref, wo_a_weight)
@triton.jit
def _mxfp8_wo_a_bmm_kernel(
a_ptr, # [T, G * K] e4m3
as_ptr, # [T, G * K // 32] E8M0
w_ptr, # [G * R, K] e4m3
ws_ptr, # [G * R, K // 32] E8M0
out_ptr, # [T, G * R]
num_tokens,
stride_at,
stride_ast,
stride_wn,
stride_wsn,
stride_out,
R: tl.constexpr,
K: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
):
# Output columns are group-major, so a BLOCK_N tile never straddles two
# groups (R % BLOCK_N == 0) and picks its group's K slice of A.
pid_n = tl.program_id(0)
pid_m = tl.program_id(1)
group = (pid_n * BLOCK_N) // R
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
m_mask = offs_m < num_tokens
offs_k = tl.arange(0, BLOCK_K)
offs_sk = tl.arange(0, BLOCK_K // 32)
a_ptrs = a_ptr + offs_m[:, None] * stride_at + (group * K + offs_k)[None, :]
as_ptrs = (
as_ptr + offs_m[:, None] * stride_ast + (group * (K // 32) + offs_sk)[None, :]
)
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :]
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for _ in range(K // BLOCK_K):
a = tl.load(a_ptrs, mask=m_mask[:, None], other=0.0)
a_s = tl.load(as_ptrs, mask=m_mask[:, None], other=127)
w = tl.load(w_ptrs)
w_s = tl.load(ws_ptrs)
acc += tl.dot_scaled(a, a_s, "e4m3", w.T, w_s, "e4m3")
a_ptrs += BLOCK_K
as_ptrs += BLOCK_K // 32
w_ptrs += BLOCK_K
ws_ptrs += BLOCK_K // 32
tl.store(
out_ptr + offs_m[:, None] * stride_out + offs_n[None, :],
acc.to(out_ptr.dtype.element_ty),
mask=m_mask[:, None],
)
def _mxfp8_wo_a_bmm_config(num_tokens: int, n_groups: int) -> tuple[int, ...]:
"""(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) for gfx950.
Tuned under HIP graphs with a cold weight at G = 4 and 2, over every
decode shape of conc 1-128 x 0-5 spec tokens plus prefill chunks up to
8K tokens. The best tile tracks the total work T * G, so the tiers are
keyed on it.
This will be replaced after new GEMM kernel from AITER with proper 32x32 scale
shape GEMM fp8 enabled.
"""
work = num_tokens * n_groups
if work <= 64:
return 16, 16, 1024, 2, 3
if work <= 128:
return 32, 16, 1024, 2, 3
if work <= 256:
return 32, 32, 512, 2, 3
if work <= 512:
return 64, 32, 512, 2, 3
if work <= 1024:
return 64, 64, 512, 4, 2
if work <= 2048:
return 64, 64, 256, 4, 2
if work <= 3072:
return 64, 64, 256, 2, 1
if work <= 4096:
return 128, 128, 256, 8, 2
return 128, 128, 128, 4, 2
def _rocm_mxfp8_wo_a_bmm_impl(
a: torch.Tensor,
a_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
n_groups: int,
o_lora_rank: int,
) -> torch.Tensor:
num_tokens = a.shape[0]
group_dim = a.shape[1] // n_groups
out = torch.empty(
(num_tokens, n_groups * o_lora_rank), dtype=torch.bfloat16, device=a.device
)
if num_tokens == 0:
return out
block_m, block_n, block_k, num_warps, num_stages = _mxfp8_wo_a_bmm_config(
num_tokens, n_groups
)
grid = (n_groups * o_lora_rank // block_n, triton.cdiv(num_tokens, block_m))
_mxfp8_wo_a_bmm_kernel[grid](
a,
a_scale,
weight,
weight_scale,
out,
num_tokens,
a.stride(0),
a_scale.stride(0),
weight.stride(0),
weight_scale.stride(0),
out.stride(0),
R=o_lora_rank,
K=group_dim,
BLOCK_M=block_m,
BLOCK_N=block_n,
BLOCK_K=block_k,
num_warps=num_warps,
num_stages=num_stages,
)
return out
def _rocm_mxfp8_wo_a_bmm_fake(
a: torch.Tensor,
a_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
n_groups: int,
o_lora_rank: int,
) -> torch.Tensor:
return a.new_empty((a.shape[0], n_groups * o_lora_rank), dtype=torch.bfloat16)
# An opaque op: the tile choice branches on the token count, which the
# compiled o_proj must not freeze at its trace-time value.
direct_register_custom_op(
op_name="rocm_dsv41_mxfp8_wo_a_bmm",
op_func=_rocm_mxfp8_wo_a_bmm_impl,
fake_impl=_rocm_mxfp8_wo_a_bmm_fake,
)
def rocm_mxfp8_wo_a_bmm(
a: torch.Tensor,
a_scale: torch.Tensor,
wo_a: torch.nn.Module,
n_groups: int,
o_lora_rank: int,
) -> torch.Tensor:
"""Grouped MXFP8 wo_a: ``out[t, g, :] = a[t, g, :] @ W[g].T``, bf16 out.
``a`` is the [T, G * K] e4m3 attention output and ``a_scale`` its
[T, G * K // 32] E8M0 scales, as the sparse decode reduce writes them.
The weight is the checkpoint's MXFP8 ``wo_a`` as loaded, [G * R, K] with
[G * R, K // 32] scales, so there is no dequantized copy to keep.
Returns [T, G * R].
"""
return torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(
a, a_scale, wo_a.weight, wo_a.weight_scale, n_groups, o_lora_rank
)
_DSV4_SPARSE_NOPE_DIM = 448 _DSV4_SPARSE_NOPE_DIM = 448
_DSV4_SPARSE_ROPE_DIM = 64 _DSV4_SPARSE_ROPE_DIM = 64
@@ -2922,6 +3175,33 @@ def _sparse_attn_decode_gfx950_partial_kernel(
) )
@triton.jit
def _mxfp8_scale_bits(amax):
"""Biased E8M0 exponent that puts ``amax`` at the top of the e4m3 range.
Same rounding as ``mxfp8_e4m3_quantize``, so the output is bit-identical to
quantizing the tensor there.
"""
amax = tl.maximum(amax, 1.1754943508222875e-38)
bits = tl.ceil(tl.log2(amax / 448.0)) + 127.0
return tl.minimum(tl.maximum(bits, 0.0), 254.0)
@triton.jit
def _mxfp8_quantize_rows(x, ROWS: tl.constexpr, COLS: tl.constexpr):
"""MXFP8-quantize ``x`` [ROWS, COLS] in registers, one scale per 32 lanes.
Returns the rescaled fp32 values (to be cast to e4m3 on store) and the
[ROWS, COLS // 32] biased E8M0 exponents.
"""
blocks = tl.reshape(x, (ROWS, COLS // 32, 32))
bits = _mxfp8_scale_bits(tl.max(tl.abs(blocks), axis=2))
# Multiply by the reciprocal: a divisor of 2**-127 would be subnormal and
# flush to zero, turning an all-zero block into NaN.
q = blocks * tl.exp2(127.0 - bits)[:, :, None]
return tl.reshape(q, (ROWS, COLS)), bits
@triton.jit @triton.jit
def _sparse_attn_decode_reduce_kernel( def _sparse_attn_decode_reduce_kernel(
part_m_ptr, part_m_ptr,
@@ -2931,8 +3211,11 @@ def _sparse_attn_decode_reduce_kernel(
out_ptr, out_ptr,
pos_ptr, pos_ptr,
cos_sin_ptr, cos_sin_ptr,
out_scale_ptr,
out_stride0, out_stride0,
out_stride1, out_stride1,
os_stride0,
os_stride1,
pm_stride0, pm_stride0,
pm_stride_s, pm_stride_s,
pa_stride0, pa_stride0,
@@ -2949,6 +3232,7 @@ def _sparse_attn_decode_reduce_kernel(
FUSE_INV_ROPE: tl.constexpr, FUSE_INV_ROPE: tl.constexpr,
NOPE: tl.constexpr, NOPE: tl.constexpr,
HALF: tl.constexpr, HALF: tl.constexpr,
QUANT_OUT: tl.constexpr,
): ):
query_idx = tl.program_id(0) query_idx = tl.program_id(0)
pid_h = tl.program_id(1) pid_h = tl.program_id(1)
@@ -3061,9 +3345,22 @@ def _sparse_attn_decode_reduce_kernel(
out_row_ptr = ( out_row_ptr = (
out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1 out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1
) )
if QUANT_OUT:
# Emit wo_a's MXFP8 input directly instead of a bf16 row that a
# separate pass would read back to quantize.
out, scale_bits = _mxfp8_quantize_rows(out, BLOCK_H, COMB_DIM)
scale_offsets = tl.arange(0, COMB_DIM // 32)
tl.store(
out_scale_ptr
+ query_idx * os_stride0
+ head_offsets[:, None] * os_stride1
+ scale_offsets[None, :],
scale_bits.to(tl.uint8),
mask=head_mask[:, None],
)
tl.store( tl.store(
out_row_ptr + comb_offsets[None, :], out_row_ptr + comb_offsets[None, :],
out, out.to(out_ptr.dtype.element_ty),
mask=head_mask[:, None], mask=head_mask[:, None],
) )
@@ -3380,7 +3677,14 @@ def _rocm_sparse_attn_decode_ragged_triton(
adaptive_splits: bool = False, adaptive_splits: bool = False,
inv_rope_positions: torch.Tensor | None = None, inv_rope_positions: torch.Tensor | None = None,
inv_rope_cos_sin_cache: torch.Tensor | None = None, inv_rope_cos_sin_cache: torch.Tensor | None = None,
out_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor: ) -> torch.Tensor:
"""Split-K sparse decode; returns the attention output.
With ``out_mxfp8 = (data, scale)`` the reduce writes MXFP8 instead of
bf16: ``data`` is [b, h * d] e4m3 and ``scale`` [b, h * d // 32] E8M0, and
``data`` viewed as [b, h, d] is returned.
"""
assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}" assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}"
assert main_cache.ndim == 3, ( assert main_cache.ndim == 3, (
f"expected main_cache=[blocks,block,bytes], got {main_cache.shape}" f"expected main_cache=[blocks,block,bytes], got {main_cache.shape}"
@@ -3445,7 +3749,28 @@ def _rocm_sparse_attn_decode_ragged_triton(
extra_indptr = torch.zeros(num_queries + 1, device=q.device, dtype=torch.int32) extra_indptr = torch.zeros(num_queries + 1, device=q.device, dtype=torch.int32)
block_h = 16 block_h = 16
if out is None: out_scale = None
if out_mxfp8 is not None:
assert out is None, "out and out_mxfp8 are mutually exclusive"
assert _ON_GFX950, "the MXFP8 reduce epilogue is gfx950-only"
assert inv_rope_positions is not None, (
"the MXFP8 output feeds wo_a, so it must be inverse-RoPE'd first"
)
out_data, out_scale = out_mxfp8
assert out_data.dtype == torch.float8_e4m3fn and out_scale.dtype == (
torch.uint8
), f"expected e4m3/uint8 MXFP8 buffers, got {out_data.dtype}/{out_scale.dtype}"
assert out_data.shape == (num_queries, num_heads * head_dim), (
f"expected MXFP8 data [{num_queries}, {num_heads * head_dim}], "
f"got {tuple(out_data.shape)}"
)
assert out_scale.shape == (num_queries, num_heads * head_dim // 32), (
f"expected MXFP8 scale [{num_queries}, {num_heads * head_dim // 32}], "
f"got {tuple(out_scale.shape)}"
)
assert out_data.stride(-1) == 1 and out_scale.stride(-1) == 1
out = out_data.view(num_queries, num_heads, head_dim)
elif out is None:
out = torch.empty_like(q, dtype=torch.bfloat16) out = torch.empty_like(q, dtype=torch.bfloat16)
else: else:
assert out.shape == q.shape, f"expected out shape {q.shape}, got {out.shape}" assert out.shape == q.shape, f"expected out shape {q.shape}, got {out.shape}"
@@ -3643,8 +3968,11 @@ def _rocm_sparse_attn_decode_ragged_triton(
out, out,
inv_rope_positions, inv_rope_positions,
inv_rope_cos_sin_cache, inv_rope_cos_sin_cache,
out_scale,
out.stride(0), out.stride(0),
out.stride(1), out.stride(1),
out_scale.stride(0) if out_scale is not None else 0,
head_dim // 32,
part_m.stride(0), part_m.stride(0),
part_m.stride(1), part_m.stride(1),
part_acc.stride(0), part_acc.stride(0),
@@ -3661,6 +3989,7 @@ def _rocm_sparse_attn_decode_ragged_triton(
FUSE_INV_ROPE=fuse_inv_rope, FUSE_INV_ROPE=fuse_inv_rope,
NOPE=nope_head_dim, NOPE=nope_head_dim,
HALF=rope_head_dim // 2, HALF=rope_head_dim // 2,
QUANT_OUT=out_scale is not None,
num_warps=4, num_warps=4,
) )
return out return out
@@ -3687,6 +4016,7 @@ def _rocm_sparse_attn_decode_triton(
adaptive_splits: bool = False, adaptive_splits: bool = False,
inv_rope_positions: torch.Tensor | None = None, inv_rope_positions: torch.Tensor | None = None,
inv_rope_cos_sin_cache: torch.Tensor | None = None, inv_rope_cos_sin_cache: torch.Tensor | None = None,
out_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor: ) -> torch.Tensor:
if main_ragged_indices is None or main_ragged_indptr is None: if main_ragged_indices is None or main_ragged_indptr is None:
main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense( main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense(
@@ -3727,6 +4057,7 @@ def _rocm_sparse_attn_decode_triton(
adaptive_splits=adaptive_splits, adaptive_splits=adaptive_splits,
inv_rope_positions=inv_rope_positions, inv_rope_positions=inv_rope_positions,
inv_rope_cos_sin_cache=inv_rope_cos_sin_cache, inv_rope_cos_sin_cache=inv_rope_cos_sin_cache,
out_mxfp8=out_mxfp8,
) )
@@ -3825,11 +4156,12 @@ def rocm_sparse_attn_decode(
head_dim: int, head_dim: int,
nope_head_dim: int, nope_head_dim: int,
rope_head_dim: int, rope_head_dim: int,
output: torch.Tensor, output: torch.Tensor | None,
extra_cache_nan_free: bool = False, extra_cache_nan_free: bool = False,
adaptive_splits: bool = False, adaptive_splits: bool = False,
inv_rope_positions: torch.Tensor | None = None, inv_rope_positions: torch.Tensor | None = None,
inv_rope_cos_sin_cache: torch.Tensor | None = None, inv_rope_cos_sin_cache: torch.Tensor | None = None,
output_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> int: ) -> int:
"""Run sparse MLA decode into ``output``. """Run sparse MLA decode into ``output``.
@@ -3837,6 +4169,11 @@ def rocm_sparse_attn_decode(
epilogue. Returns how many leading rows of ``output`` came back rotated, epilogue. Returns how many leading rows of ``output`` came back rotated,
so a caller mixing in a decode path that does not fuse still knows what it so a caller mixing in a decode path that does not fuse still knows what it
owes the standalone pass. Read it from the eager attention segment only. owes the standalone pass. Read it from the eager attention segment only.
``output_mxfp8 = (data, scale)`` replaces ``output``: the reduce also
MXFP8-quantizes the rotated rows for the FP8 wo_a (see
``_rocm_sparse_attn_decode_ragged_triton``). It needs gfx950 and the fused
inverse RoPE, and always covers every row.
""" """
assert swa_k_cache.dtype == torch.uint8, ( assert swa_k_cache.dtype == torch.uint8, (
"ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, " "ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, "
@@ -3866,7 +4203,12 @@ def rocm_sparse_attn_decode(
if topk_indices is not None: if topk_indices is not None:
extra_indices = topk_indices.reshape(topk_indices.shape[0], -1) extra_indices = topk_indices.reshape(topk_indices.shape[0], -1)
direct_out = output if _ON_GFX950 and output.dtype == torch.bfloat16 else None if output_mxfp8 is not None:
assert output is None, "output and output_mxfp8 are mutually exclusive"
direct_out = None
else:
assert output is not None
direct_out = output if _ON_GFX950 and output.dtype == torch.bfloat16 else None
attn_out = _rocm_sparse_attn_decode_triton( attn_out = _rocm_sparse_attn_decode_triton(
q=q, q=q,
main_cache=swa_k_cache, main_cache=swa_k_cache,
@@ -3888,7 +4230,11 @@ def rocm_sparse_attn_decode(
adaptive_splits=adaptive_splits, adaptive_splits=adaptive_splits,
inv_rope_positions=inv_rope_positions, inv_rope_positions=inv_rope_positions,
inv_rope_cos_sin_cache=inv_rope_cos_sin_cache, inv_rope_cos_sin_cache=inv_rope_cos_sin_cache,
out_mxfp8=output_mxfp8,
) )
if output_mxfp8 is not None:
return q.shape[0]
assert output is not None
if direct_out is None: if direct_out is None:
output.copy_(attn_out.to(output.dtype)) output.copy_(attn_out.to(output.dtype))
return output.shape[0] if inv_rope_positions is not None else 0 return output.shape[0] if inv_rope_positions is not None else 0