[Minimax-M3] Keep the native FP8 MMA in the Triton indexer scorers (#59481)

Signed-off-by: Rajas Mhaskar <rmhaskar@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
rmhaskar
2026-10-02 12:30:39 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 167a81fa9c
commit e42d35d4f3
2 changed files with 45 additions and 9 deletions
@@ -124,9 +124,11 @@ def _index_block_score_kernel(
order=(1, 0),
)
q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
# FP8 has no mixed-dtype dot with bf16/fp32. Leave bf16 and fp32 loads
# in their stored dtype so fp32 pipeline tests keep full precision.
if q.dtype.is_fp8():
# A matched fp8 pair (e.g. e4m3 x e4m3) lowers to a native FP8 MMA, so
# upcast only when the operands differ: there is no mixed-dtype fp8 MMA,
# including across fp8 flavours (fp8e4nv vs fp8e4b8). bf16 and fp32 loads
# always stay in their stored dtype.
if q.dtype.is_fp8() and q.dtype != ik_cache_ptr.dtype.element_ty:
q = q.to(tl.bfloat16)
q_start = prefix_len + pid_q * BLOCK_SIZE_Q
@@ -162,7 +164,7 @@ def _index_block_score_kernel(
+ off_k[None, :] * stride_ik_pos
+ off_d[:, None] * stride_ik_d,
)
if k.dtype.is_fp8():
if k.dtype.is_fp8() and k.dtype != q.dtype:
k = k.to(tl.bfloat16)
qk = tl.dot(q, k, out_dtype=tl.float32)
# apply causal mask as needed
@@ -378,7 +380,7 @@ def _decode_index_score_kernel(
mask=q_mask[None, :],
other=0.0,
) # [D,HQ]
if q.dtype.is_fp8():
if q.dtype.is_fp8() and q.dtype != ik_cache_ptr.dtype.element_ty:
q = q.to(tl.bfloat16)
for blk in tl.range(chunk_start_block, chunk_end_block):
page = tl.load(bt_row + blk).to(tl.int64)
@@ -393,10 +395,10 @@ def _decode_index_score_kernel(
+ off_k[:, None] * stride_ik_pos
+ off_d * stride_ik_d,
) # [N,D]
# Upcast only FP8 cache loads. BF16 operands keep FP8 dot
# instructions out of the fallback path; FP32 accumulation
# preserves score accuracy. BF16/FP32 loads stay as stored.
if k.dtype.is_fp8():
# Matched fp8 operands keep the native FP8 MMA; any mismatch upcasts,
# since no mixed-dtype fp8 MMA exists. FP32 accumulation preserves
# score accuracy either way. BF16/FP32 loads stay as stored.
if k.dtype.is_fp8() and k.dtype != q.dtype:
k = k.to(tl.bfloat16)
kq = tl.dot(k, q, out_dtype=tl.float32) # [N,HQ]
kq = tl.where(pos_mask & q_mask[None, :], kq, float("-inf"))