[Perf][MiniMax-M3] MSA prefill attends with the E4M3 query; bump fmha_sm100 to 968061a (#60684)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f8471164bf
commit
2eb491e18b
@@ -614,7 +614,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
|
||||
q = qkv.new_empty((num_tokens, self.q_size))
|
||||
# With query_fp8, q is emitted only in fp8; the attend dequantizes it
|
||||
# into ``q`` for the tokens that run on bf16-query kernels.
|
||||
# into ``q`` for the tokens that run on Triton decode.
|
||||
query_fp8 = self._allocate_query_fp8(qkv)
|
||||
# index_q matches the index-K cache dtype (e4m3 for the fp8 score path);
|
||||
# the fused kernel emits fp8 directly when this buffer is e4m3.
|
||||
|
||||
@@ -266,7 +266,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
q = query[:num_tokens].view(-1, self.num_heads, hd)
|
||||
out = output[:num_tokens].view(-1, self.num_heads, hd)
|
||||
# Given query_fp8, it is the only q the fused insert wrote: CUTLASS
|
||||
# decode reads it directly, and the bf16-query kernels read it
|
||||
# decode and prefill read it directly, and Triton decode reads it
|
||||
# dequantized into ``query``.
|
||||
q_fp8 = (
|
||||
None
|
||||
@@ -339,11 +339,11 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
# [H, prefill, MK] transposed view; build_k2q_csr consumes the
|
||||
# strided view directly (topK stays innermost-contiguous).
|
||||
prefill_topk = topk[nd:num_tokens].transpose(0, 1)
|
||||
qp = (
|
||||
q[nd:]
|
||||
if q_fp8 is None
|
||||
else _dequantize_query(q_fp8[nd:], q_scale_float, q[nd:])
|
||||
)
|
||||
# An E4M3 query attends as is, its scale folded into softmax_scale.
|
||||
if q_fp8 is None:
|
||||
qp, softmax_scale = q[nd:], self.scale
|
||||
else:
|
||||
qp, softmax_scale = q_fp8[nd:], self.scale * q_scale_float
|
||||
k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr(
|
||||
prefill_topk,
|
||||
p.cu_seqlens_q,
|
||||
@@ -375,7 +375,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
max_seqlen_k=p.max_seq_len,
|
||||
blk_kv=SPARSE_BLOCK_SIZE,
|
||||
causal=True,
|
||||
softmax_scale=self.scale,
|
||||
softmax_scale=softmax_scale,
|
||||
page_table=p.block_table,
|
||||
seqused_k=p.seq_lens,
|
||||
schedule=schedule,
|
||||
@@ -393,7 +393,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl):
|
||||
topK=self.topk_blocks,
|
||||
blk_kv=SPARSE_BLOCK_SIZE,
|
||||
causal=True,
|
||||
softmax_scale=self.scale,
|
||||
softmax_scale=softmax_scale,
|
||||
cu_seqlens_q=p.cu_seqlens_q,
|
||||
cu_seqlens_k=p.cu_seqlens_k,
|
||||
max_seqlen_q=p.max_query_len,
|
||||
|
||||
Reference in New Issue
Block a user