[Minimax M3] Enable fp8 indexer cache on triton indexer for non-SM100 architectures. (#59081)

Signed-off-by: Jarrel Seah <jarrel@harrison.ai>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Signed-off-by: Rajas Mhaskar <rmhaskar@nvidia.com>
Co-authored-by: Jarrel Seah <jarrel@harrison.ai>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
rmhaskar
2026-09-28 19:21:13 -04:00
committed by GitHub
co-authored by Jarrel Seah Tyler Michael Smith OpenAI Codex
parent 6a6c2952fa
commit e03875f4d2
3 changed files with 133 additions and 9 deletions
+10 -5
View File
@@ -502,16 +502,21 @@ def select_indexer_impl_cls(
On Blackwell (SM100) with ``topk_blocks == 16`` (the only width fmha_sm100's
``sparse_topk_select`` kernel supports), the fmha_sm100 score + top-k path is
used for both bf16 and fp8 index caches. Everything else falls back to the
Triton indexer (bf16 only).
Triton indexer, with fp8 restricted to CUDA platforms that advertise fp8
support.
"""
if indexer_kv_dtype in ("mxfp4", "nvfp4"):
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} needs the (not-yet-added) "
"CuteDSL indexer impl."
)
is_sm100 = (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
)
is_cuda = current_platform.is_cuda()
is_fp8 = indexer_kv_dtype in ("fp8", "fp8_e4m3")
if is_fp8 and not (is_cuda and current_platform.supports_fp8()):
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} requires CUDA fp8 support."
)
is_sm100 = is_cuda and current_platform.is_device_capability_family(100)
use_msa = (
is_sm100
and topk_blocks == 16
@@ -530,7 +535,7 @@ def select_indexer_impl_cls(
indexer_kv_dtype,
)
return MiniMaxM3IndexerMSAImpl
if indexer_kv_dtype != "bf16":
if indexer_kv_dtype not in ("bf16", "fp8", "fp8_e4m3"):
raise NotImplementedError(
f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported by the "
"Triton indexer impl."
@@ -123,6 +123,10 @@ def _index_block_score_kernel(
order=(1, 0),
)
q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
# FP8 has no mixed-dtype dot with bf16/fp32. Leave bf16 and fp32 loads
# in their stored dtype so fp32 pipeline tests keep full precision.
if q.dtype.is_fp8():
q = q.to(tl.bfloat16)
q_start = prefix_len + pid_q * BLOCK_SIZE_Q
off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len
@@ -146,7 +150,9 @@ def _index_block_score_kernel(
+ off_k[None, :] * stride_ik_pos
+ off_d[:, None] * stride_ik_d,
)
qk = tl.dot(q, k)
if k.dtype.is_fp8():
k = k.to(tl.bfloat16)
qk = tl.dot(q, k, out_dtype=tl.float32)
# apply causal mask as needed
if q_start < i + BLOCK_SIZE_K:
qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf"))
@@ -360,6 +366,8 @@ def _decode_index_score_kernel(
mask=q_mask[None, :],
other=0.0,
) # [D,HQ]
if q.dtype.is_fp8():
q = q.to(tl.bfloat16)
for blk in tl.range(chunk_start_block, chunk_end_block):
page = tl.load(bt_row + blk).to(tl.int64)
pos = blk * BLOCK_SIZE_K + off_k
@@ -373,9 +381,11 @@ def _decode_index_score_kernel(
+ off_k[:, None] * stride_ik_pos
+ off_d * stride_ik_d,
) # [N,D]
# fp32 accumulation is required for the fp8 (e4m3) index cache: q/k are
# loaded in their stored dtype (bf16 or e4m3) and the MMA accumulates in
# fp32 so the per-block max score is exact for the fp8 indexer too.
# Upcast only FP8 cache loads. BF16 operands keep FP8 dot
# instructions out of the fallback path; FP32 accumulation
# preserves score accuracy. BF16/FP32 loads stay as stored.
if k.dtype.is_fp8():
k = k.to(tl.bfloat16)
kq = tl.dot(k, q, out_dtype=tl.float32) # [N,HQ]
kq = tl.where(pos_mask & q_mask[None, :], kq, float("-inf"))
score = tl.max(kq, axis=0) # [HQ]