[Minimax M3] Enable fp8 indexer cache on triton indexer for non-SM100 architectures. (#59081)

Signed-off-by: Jarrel Seah <jarrel@harrison.ai>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Rajas Mhaskar <rmhaskar@nvidia.com>
Co-authored-by: Jarrel Seah <jarrel@harrison.ai>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
rmhaskar
2026-09-28 19:21:13 -04:00
committed by GitHub
co-authored by Jarrel Seah Tyler Michael Smith OpenAI Codex
parent 6a6c2952fa
commit e03875f4d2
3 changed files with 133 additions and 9 deletions
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""fp8 query and index-K cache on the Triton indexer must agree with bf16.
The MiniMax M3 lightning-indexer scores only feed a top-k block ranking (the
attention itself reads the main KV cache), so an fp8 (e4m3) side cache trades
~2% relative score error for half the per-step index-K read bandwidth — the
dominant linear-in-context decode cost at long context. #45892 enabled fp8
index caches on the SM100 MSA path; this covers the Triton impl used
everywhere else (e.g. SM120), for both the decode and prefill scorers.
"""
import pytest
import torch
from vllm.models.minimax_m3.common.ops.index_topk import (
SPARSE_BLOCK_SIZE,
minimax_m3_index_decode,
minimax_m3_index_score,
minimax_m3_index_topk,
)
from vllm.platforms import current_platform
FP8_CUDA_SUPPORTED = current_platform.is_cuda() and current_platform.supports_fp8()
PAGE = SPARSE_BLOCK_SIZE
HEAD_DIM = 128
HEADS = 1
TOPK = 16
NUM_PAGES = 100
SEQ_LEN = 96 * PAGE + 37
PLANTED = (7, 40, 88)
def _make_inputs(device: str) -> tuple[torch.Tensor, torch.Tensor]:
torch.manual_seed(3)
keys = torch.randn(NUM_PAGES * PAGE, HEAD_DIM, dtype=torch.bfloat16, device=device)
# RMS-normalize like the real (post index_k_norm) keys.
keys = keys * keys.float().pow(2).mean(-1, keepdim=True).add(1e-6).rsqrt().to(
torch.bfloat16
)
q = torch.randn(1, HEADS, HEAD_DIM, dtype=torch.bfloat16, device=device)
for b in PLANTED:
keys[b * PAGE + 5] = (q[0, 0] * 3).to(torch.bfloat16)
return q, keys
def _check_selection(top_bf16: set[int], top_fp8: set[int]) -> None:
# Planted (clearly relevant) blocks must never be lost.
assert all(b in top_fp8 for b in PLANTED)
# Selection may differ only at the noise floor (near-tied filler blocks).
assert len(top_bf16 & top_fp8) >= TOPK - 2
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_fp8_index_query_and_cache_decode_topk_matches_bf16() -> None:
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
query = q.to(cache.dtype)
idx = minimax_m3_index_decode(
query,
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
seq_lens,
SEQ_LEN,
TOPK,
0,
0,
HEADS,
1,
1,
)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
# single decode-like query token at the end of the sequence
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=dev)
prefix_lens = torch.tensor([SEQ_LEN - 1], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
query = q.to(cache.dtype)
score = minimax_m3_index_score(
query,
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
cu_seqlens_q,
seq_lens,
prefix_lens,
1,
SEQ_LEN,
HEADS,
)
idx = minimax_m3_index_topk(score, cu_seqlens_q, prefix_lens, 1, TOPK, 0, 0)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
+10 -5
View File
@@ -502,16 +502,21 @@ def select_indexer_impl_cls(
On Blackwell (SM100) with ``topk_blocks == 16`` (the only width fmha_sm100's
``sparse_topk_select`` kernel supports), the fmha_sm100 score + top-k path is
used for both bf16 and fp8 index caches. Everything else falls back to the
Triton indexer (bf16 only).
Triton indexer, with fp8 restricted to CUDA platforms that advertise fp8
support.
"""
if indexer_kv_dtype in ("mxfp4", "nvfp4"):
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} needs the (not-yet-added) "
"CuteDSL indexer impl."
)
is_sm100 = (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
)
is_cuda = current_platform.is_cuda()
is_fp8 = indexer_kv_dtype in ("fp8", "fp8_e4m3")
if is_fp8 and not (is_cuda and current_platform.supports_fp8()):
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} requires CUDA fp8 support."
)
is_sm100 = is_cuda and current_platform.is_device_capability_family(100)
use_msa = (
is_sm100
and topk_blocks == 16
@@ -530,7 +535,7 @@ def select_indexer_impl_cls(
indexer_kv_dtype,
)
return MiniMaxM3IndexerMSAImpl
if indexer_kv_dtype != "bf16":
if indexer_kv_dtype not in ("bf16", "fp8", "fp8_e4m3"):
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported by the "
"Triton indexer impl."
@@ -123,6 +123,10 @@ def _index_block_score_kernel(
order=(1, 0),
)
q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
# FP8 has no mixed-dtype dot with bf16/fp32. Leave bf16 and fp32 loads
# in their stored dtype so fp32 pipeline tests keep full precision.
if q.dtype.is_fp8():
q = q.to(tl.bfloat16)
q_start = prefix_len + pid_q * BLOCK_SIZE_Q
off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len
@@ -146,7 +150,9 @@ def _index_block_score_kernel(
+ off_k[None, :] * stride_ik_pos
+ off_d[:, None] * stride_ik_d,
)
qk = tl.dot(q, k)
if k.dtype.is_fp8():
k = k.to(tl.bfloat16)
qk = tl.dot(q, k, out_dtype=tl.float32)
# apply causal mask as needed
if q_start < i + BLOCK_SIZE_K:
qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf"))
@@ -360,6 +366,8 @@ def _decode_index_score_kernel(
mask=q_mask[None, :],
other=0.0,
) # [D,HQ]
if q.dtype.is_fp8():
q = q.to(tl.bfloat16)
for blk in tl.range(chunk_start_block, chunk_end_block):
page = tl.load(bt_row + blk).to(tl.int64)
pos = blk * BLOCK_SIZE_K + off_k
@@ -373,9 +381,11 @@ def _decode_index_score_kernel(
+ off_k[:, None] * stride_ik_pos
+ off_d * stride_ik_d,
) # [N,D]
# fp32 accumulation is required for the fp8 (e4m3) index cache: q/k are
# loaded in their stored dtype (bf16 or e4m3) and the MMA accumulates in
# fp32 so the per-block max score is exact for the fp8 indexer too.
# Upcast only FP8 cache loads. BF16 operands keep FP8 dot
# instructions out of the fallback path; FP32 accumulation
# preserves score accuracy. BF16/FP32 loads stay as stored.
if k.dtype.is_fp8():
k = k.to(tl.bfloat16)
kq = tl.dot(k, q, out_dtype=tl.float32) # [N,HQ]
kq = tl.where(pos_mask & q_mask[None, :], kq, float("-inf"))
score = tl.max(kq, axis=0) # [HQ]