diff --git a/tests/kernels/attention/test_minimax_m3_fp8_triton_indexer.py b/tests/kernels/attention/test_minimax_m3_fp8_triton_indexer.py new file mode 100644 index 0000000000..247faba17f --- /dev/null +++ b/tests/kernels/attention/test_minimax_m3_fp8_triton_indexer.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""fp8 query and index-K cache on the Triton indexer must agree with bf16. + +The MiniMax M3 lightning-indexer scores only feed a top-k block ranking (the +attention itself reads the main KV cache), so an fp8 (e4m3) side cache trades +~2% relative score error for half the per-step index-K read bandwidth — the +dominant linear-in-context decode cost at long context. #45892 enabled fp8 +index caches on the SM100 MSA path; this covers the Triton impl used +everywhere else (e.g. SM120), for both the decode and prefill scorers. +""" + +import pytest +import torch + +from vllm.models.minimax_m3.common.ops.index_topk import ( + SPARSE_BLOCK_SIZE, + minimax_m3_index_decode, + minimax_m3_index_score, + minimax_m3_index_topk, +) +from vllm.platforms import current_platform + +FP8_CUDA_SUPPORTED = current_platform.is_cuda() and current_platform.supports_fp8() +PAGE = SPARSE_BLOCK_SIZE +HEAD_DIM = 128 +HEADS = 1 +TOPK = 16 +NUM_PAGES = 100 +SEQ_LEN = 96 * PAGE + 37 +PLANTED = (7, 40, 88) + + +def _make_inputs(device: str) -> tuple[torch.Tensor, torch.Tensor]: + torch.manual_seed(3) + keys = torch.randn(NUM_PAGES * PAGE, HEAD_DIM, dtype=torch.bfloat16, device=device) + # RMS-normalize like the real (post index_k_norm) keys. + keys = keys * keys.float().pow(2).mean(-1, keepdim=True).add(1e-6).rsqrt().to( + torch.bfloat16 + ) + q = torch.randn(1, HEADS, HEAD_DIM, dtype=torch.bfloat16, device=device) + for b in PLANTED: + keys[b * PAGE + 5] = (q[0, 0] * 3).to(torch.bfloat16) + return q, keys + + +def _check_selection(top_bf16: set[int], top_fp8: set[int]) -> None: + # Planted (clearly relevant) blocks must never be lost. + assert all(b in top_fp8 for b in PLANTED) + # Selection may differ only at the noise floor (near-tied filler blocks). + assert len(top_bf16 & top_fp8) >= TOPK - 2 + + +@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required") +@torch.inference_mode() +def test_fp8_index_query_and_cache_decode_topk_matches_bf16() -> None: + dev = "cuda" + q, keys = _make_inputs(dev) + block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0) + seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev) + + def select(cache: torch.Tensor) -> set[int]: + query = q.to(cache.dtype) + idx = minimax_m3_index_decode( + query, + cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(), + block_table, + seq_lens, + SEQ_LEN, + TOPK, + 0, + 0, + HEADS, + 1, + 1, + ) + return {int(b) for b in idx[0, 0].tolist() if b >= 0} + + _check_selection(select(keys), select(keys.to(torch.float8_e4m3fn))) + + +@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required") +@torch.inference_mode() +def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None: + dev = "cuda" + q, keys = _make_inputs(dev) + block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0) + seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev) + # single decode-like query token at the end of the sequence + cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=dev) + prefix_lens = torch.tensor([SEQ_LEN - 1], dtype=torch.int32, device=dev) + + def select(cache: torch.Tensor) -> set[int]: + query = q.to(cache.dtype) + score = minimax_m3_index_score( + query, + cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(), + block_table, + cu_seqlens_q, + seq_lens, + prefix_lens, + 1, + SEQ_LEN, + HEADS, + ) + idx = minimax_m3_index_topk(score, cu_seqlens_q, prefix_lens, 1, TOPK, 0, 0) + return {int(b) for b in idx[0, 0].tolist() if b >= 0} + + _check_selection(select(keys), select(keys.to(torch.float8_e4m3fn))) diff --git a/vllm/models/minimax_m3/common/indexer.py b/vllm/models/minimax_m3/common/indexer.py index 2459d4ef85..4e8d6dbd3a 100644 --- a/vllm/models/minimax_m3/common/indexer.py +++ b/vllm/models/minimax_m3/common/indexer.py @@ -502,16 +502,21 @@ def select_indexer_impl_cls( On Blackwell (SM100) with ``topk_blocks == 16`` (the only width fmha_sm100's ``sparse_topk_select`` kernel supports), the fmha_sm100 score + top-k path is used for both bf16 and fp8 index caches. Everything else falls back to the - Triton indexer (bf16 only). + Triton indexer, with fp8 restricted to CUDA platforms that advertise fp8 + support. """ if indexer_kv_dtype in ("mxfp4", "nvfp4"): raise NotImplementedError( f"indexer_kv_dtype={indexer_kv_dtype!r} needs the (not-yet-added) " "CuteDSL indexer impl." ) - is_sm100 = ( - current_platform.is_cuda() and current_platform.is_device_capability_family(100) - ) + is_cuda = current_platform.is_cuda() + is_fp8 = indexer_kv_dtype in ("fp8", "fp8_e4m3") + if is_fp8 and not (is_cuda and current_platform.supports_fp8()): + raise NotImplementedError( + f"indexer_kv_dtype={indexer_kv_dtype!r} requires CUDA fp8 support." + ) + is_sm100 = is_cuda and current_platform.is_device_capability_family(100) use_msa = ( is_sm100 and topk_blocks == 16 @@ -530,7 +535,7 @@ def select_indexer_impl_cls( indexer_kv_dtype, ) return MiniMaxM3IndexerMSAImpl - if indexer_kv_dtype != "bf16": + if indexer_kv_dtype not in ("bf16", "fp8", "fp8_e4m3"): raise NotImplementedError( f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported by the " "Triton indexer impl." diff --git a/vllm/models/minimax_m3/common/ops/index_topk.py b/vllm/models/minimax_m3/common/ops/index_topk.py index ed677fd871..5123dabe90 100644 --- a/vllm/models/minimax_m3/common/ops/index_topk.py +++ b/vllm/models/minimax_m3/common/ops/index_topk.py @@ -123,6 +123,10 @@ def _index_block_score_kernel( order=(1, 0), ) q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero") + # FP8 has no mixed-dtype dot with bf16/fp32. Leave bf16 and fp32 loads + # in their stored dtype so fp32 pipeline tests keep full precision. + if q.dtype.is_fp8(): + q = q.to(tl.bfloat16) q_start = prefix_len + pid_q * BLOCK_SIZE_Q off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len @@ -146,7 +150,9 @@ def _index_block_score_kernel( + off_k[None, :] * stride_ik_pos + off_d[:, None] * stride_ik_d, ) - qk = tl.dot(q, k) + if k.dtype.is_fp8(): + k = k.to(tl.bfloat16) + qk = tl.dot(q, k, out_dtype=tl.float32) # apply causal mask as needed if q_start < i + BLOCK_SIZE_K: qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf")) @@ -360,6 +366,8 @@ def _decode_index_score_kernel( mask=q_mask[None, :], other=0.0, ) # [D,HQ] + if q.dtype.is_fp8(): + q = q.to(tl.bfloat16) for blk in tl.range(chunk_start_block, chunk_end_block): page = tl.load(bt_row + blk).to(tl.int64) pos = blk * BLOCK_SIZE_K + off_k @@ -373,9 +381,11 @@ def _decode_index_score_kernel( + off_k[:, None] * stride_ik_pos + off_d * stride_ik_d, ) # [N,D] - # fp32 accumulation is required for the fp8 (e4m3) index cache: q/k are - # loaded in their stored dtype (bf16 or e4m3) and the MMA accumulates in - # fp32 so the per-block max score is exact for the fp8 indexer too. + # Upcast only FP8 cache loads. BF16 operands keep FP8 dot + # instructions out of the fallback path; FP32 accumulation + # preserves score accuracy. BF16/FP32 loads stay as stored. + if k.dtype.is_fp8(): + k = k.to(tl.bfloat16) kq = tl.dot(k, q, out_dtype=tl.float32) # [N,HQ] kq = tl.where(pos_mask & q_mask[None, :], kq, float("-inf")) score = tl.max(kq, axis=0) # [HQ]