[Minimax M3] Enable fp8 indexer cache on triton indexer for non-SM100 architectures. (#59081)
Signed-off-by: Jarrel Seah <jarrel@harrison.ai> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Rajas Mhaskar <rmhaskar@nvidia.com> Co-authored-by: Jarrel Seah <jarrel@harrison.ai> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Jarrel Seah
Tyler Michael Smith
OpenAI Codex
parent
6a6c2952fa
commit
e03875f4d2
@@ -502,16 +502,21 @@ def select_indexer_impl_cls(
|
||||
On Blackwell (SM100) with ``topk_blocks == 16`` (the only width fmha_sm100's
|
||||
``sparse_topk_select`` kernel supports), the fmha_sm100 score + top-k path is
|
||||
used for both bf16 and fp8 index caches. Everything else falls back to the
|
||||
Triton indexer (bf16 only).
|
||||
Triton indexer, with fp8 restricted to CUDA platforms that advertise fp8
|
||||
support.
|
||||
"""
|
||||
if indexer_kv_dtype in ("mxfp4", "nvfp4"):
|
||||
raise NotImplementedError(
|
||||
f"indexer_kv_dtype={indexer_kv_dtype!r} needs the (not-yet-added) "
|
||||
"CuteDSL indexer impl."
|
||||
)
|
||||
is_sm100 = (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
is_cuda = current_platform.is_cuda()
|
||||
is_fp8 = indexer_kv_dtype in ("fp8", "fp8_e4m3")
|
||||
if is_fp8 and not (is_cuda and current_platform.supports_fp8()):
|
||||
raise NotImplementedError(
|
||||
f"indexer_kv_dtype={indexer_kv_dtype!r} requires CUDA fp8 support."
|
||||
)
|
||||
is_sm100 = is_cuda and current_platform.is_device_capability_family(100)
|
||||
use_msa = (
|
||||
is_sm100
|
||||
and topk_blocks == 16
|
||||
@@ -530,7 +535,7 @@ def select_indexer_impl_cls(
|
||||
indexer_kv_dtype,
|
||||
)
|
||||
return MiniMaxM3IndexerMSAImpl
|
||||
if indexer_kv_dtype != "bf16":
|
||||
if indexer_kv_dtype not in ("bf16", "fp8", "fp8_e4m3"):
|
||||
raise NotImplementedError(
|
||||
f"indexer_kv_dtype={indexer_kv_dtype!r} is not supported by the "
|
||||
"Triton indexer impl."
|
||||
|
||||
@@ -123,6 +123,10 @@ def _index_block_score_kernel(
|
||||
order=(1, 0),
|
||||
)
|
||||
q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
|
||||
# FP8 has no mixed-dtype dot with bf16/fp32. Leave bf16 and fp32 loads
|
||||
# in their stored dtype so fp32 pipeline tests keep full precision.
|
||||
if q.dtype.is_fp8():
|
||||
q = q.to(tl.bfloat16)
|
||||
q_start = prefix_len + pid_q * BLOCK_SIZE_Q
|
||||
|
||||
off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len
|
||||
@@ -146,7 +150,9 @@ def _index_block_score_kernel(
|
||||
+ off_k[None, :] * stride_ik_pos
|
||||
+ off_d[:, None] * stride_ik_d,
|
||||
)
|
||||
qk = tl.dot(q, k)
|
||||
if k.dtype.is_fp8():
|
||||
k = k.to(tl.bfloat16)
|
||||
qk = tl.dot(q, k, out_dtype=tl.float32)
|
||||
# apply causal mask as needed
|
||||
if q_start < i + BLOCK_SIZE_K:
|
||||
qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf"))
|
||||
@@ -360,6 +366,8 @@ def _decode_index_score_kernel(
|
||||
mask=q_mask[None, :],
|
||||
other=0.0,
|
||||
) # [D,HQ]
|
||||
if q.dtype.is_fp8():
|
||||
q = q.to(tl.bfloat16)
|
||||
for blk in tl.range(chunk_start_block, chunk_end_block):
|
||||
page = tl.load(bt_row + blk).to(tl.int64)
|
||||
pos = blk * BLOCK_SIZE_K + off_k
|
||||
@@ -373,9 +381,11 @@ def _decode_index_score_kernel(
|
||||
+ off_k[:, None] * stride_ik_pos
|
||||
+ off_d * stride_ik_d,
|
||||
) # [N,D]
|
||||
# fp32 accumulation is required for the fp8 (e4m3) index cache: q/k are
|
||||
# loaded in their stored dtype (bf16 or e4m3) and the MMA accumulates in
|
||||
# fp32 so the per-block max score is exact for the fp8 indexer too.
|
||||
# Upcast only FP8 cache loads. BF16 operands keep FP8 dot
|
||||
# instructions out of the fallback path; FP32 accumulation
|
||||
# preserves score accuracy. BF16/FP32 loads stay as stored.
|
||||
if k.dtype.is_fp8():
|
||||
k = k.to(tl.bfloat16)
|
||||
kq = tl.dot(k, q, out_dtype=tl.float32) # [N,HQ]
|
||||
kq = tl.where(pos_mask & q_mask[None, :], kq, float("-inf"))
|
||||
score = tl.max(kq, axis=0) # [HQ]
|
||||
|
||||
Reference in New Issue
Block a user