[Bugfix][MiniMax-M3] Keep FP8 query allocation stable across CUDA graph replay (#51203)

Signed-off-by: Kyle Liang <kylliang@nvidia.com>
This commit is contained in:
kyleliang-nv
2026-08-20 10:45:28 -07:00
committed by GitHub
parent 00f7f25828
commit 7c8b68b9ce
2 changed files with 91 additions and 11 deletions
+10 -10
View File
@@ -564,6 +564,15 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
def _allocate_query_fp8(self, qkv: torch.Tensor) -> torch.Tensor | None:
if not getattr(self.impl, "use_cutlass_decode", False):
return None
return torch.empty(
(qkv.shape[0], self.q_size),
dtype=torch.float8_e4m3fn,
device=qkv.device,
)
def forward(
self,
positions: torch.Tensor,
@@ -598,16 +607,7 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
main_slot_mapping = fwd_slot_mapping[self.layer_name]
index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix]
q = qkv.new_empty((num_tokens, self.q_size))
use_msa_decode = self.impl.should_use_msa_decode(self.layer_name)
query_fp8 = (
torch.empty(
(num_tokens, self.q_size),
dtype=torch.float8_e4m3fn,
device=qkv.device,
)
if use_msa_decode
else None
)
query_fp8 = self._allocate_query_fp8(qkv)
# index_q matches the index-K cache dtype (e4m3 for the fp8 score path);
# the fused kernel emits fp8 directly when this buffer is e4m3.
index_q = qkv.new_empty(