[Bugfix][MiniMax-M3] Fence decode indexer shared-memory stage reuse (#60194)

Signed-off-by: Vladimir Anisimov <vanisimov@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
vovani
2026-10-07 01:35:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 OpenAI Codex Thien Tran
parent 4b4eb6277a
commit fe51194e2d
2 changed files with 35 additions and 8 deletions
+32 -8
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for MiniMax M3 sparse prefill attention kernels."""
from functools import partial
import pytest
import torch
@@ -1444,19 +1446,21 @@ def test_decode_index_topk_fp8(num_idx_heads: int):
reason="CuteDSL index decode score requires Blackwell.",
)
@pytest.mark.parametrize(
("dtype", "decode_query_len", "max_decode_query_len"),
("dtype", "decode_query_len", "max_decode_query_len", "concurrent_writes"),
[
(torch.bfloat16, 1, 1),
(torch.bfloat16, 3, 8),
(torch.float8_e4m3fn, 1, 1),
(torch.float8_e4m3fn, 3, 8),
(torch.float8_e4m3fn, 8, 8),
(torch.bfloat16, 1, 1, False),
(torch.bfloat16, 3, 8, False),
(torch.float8_e4m3fn, 1, 1, False),
(torch.float8_e4m3fn, 3, 8, False),
(torch.float8_e4m3fn, 8, 8, False),
pytest.param(torch.float8_e4m3fn, 1, 1, True, id="concurrent-writes"),
],
)
def test_decode_index_score_cutedsl_correctness(
dtype: torch.dtype,
decode_query_len: int,
max_decode_query_len: int,
concurrent_writes: bool,
):
pytest.importorskip("cutlass")
from vllm.models.minimax_m3.nvidia.ops import (
@@ -1466,7 +1470,8 @@ def test_decode_index_score_cutedsl_correctness(
torch.manual_seed(0)
init_blocks, local_blocks = 0, 0
num_idx_heads, head_dim = 4, 128
active_seq_lens = torch.tensor((1025, 4097), device="cuda", dtype=torch.int32)
lengths = (131326, 131289, 131252, 131215) if concurrent_writes else (1025, 4097)
active_seq_lens = torch.tensor(lengths, device="cuda", dtype=torch.int32)
batch = active_seq_lens.numel()
total_q = batch * decode_query_len
max_seq_len = int(active_seq_lens.max())
@@ -1488,7 +1493,8 @@ def test_decode_index_score_cutedsl_correctness(
)
score = unified_score.transpose(0, 1)
minimax_m3_index_decode_score_cutedsl(
launch = partial(
minimax_m3_index_decode_score_cutedsl,
idx_q,
index_kv_cache,
block_table,
@@ -1501,6 +1507,7 @@ def test_decode_index_score_cutedsl_correctness(
max_decode_query_len=max_decode_query_len,
score_out=score,
)
launch()
expected = _reference_decode_index_score(
idx_q,
index_kv_cache,
@@ -1511,6 +1518,23 @@ def test_decode_index_score_cutedsl_correctness(
)
torch.testing.assert_close(score, expected)
if concurrent_writes:
# Concurrent stores expose shared-memory reuse before ldmatrix reads finish.
quiet = score.clone()
noise = torch.zeros(1 << 26, device="cuda")
side = torch.cuda.Stream()
side.wait_stream(torch.cuda.current_stream())
try:
for _ in range(20):
with torch.cuda.stream(side):
for _ in range(40):
noise.mul_(1.0)
launch()
torch.testing.assert_close(score, quiet, atol=0, rtol=0)
side.synchronize()
finally:
side.synchronize()
# Sparse attention kernels.
def _reference_sparse_attn(
@@ -287,6 +287,8 @@ class IndexDecodeScoreKernel:
cute.arch.barrier(barrier_id=self.BAR_MMA, number_of_threads=128)
for q in cutlass.range_constexpr(Q_TILES):
cute.copy(ldsm_atom, sQ_ldsm[None, (q, None)], rQ[None, None, q])
# Order ldmatrix reads before the producer refills this stage with TMA.
cute.arch.fence_proxy("async.shared", space="cta")
cute.arch.mbarrier_arrive(tma_empty_mbar)
tma_stage = 1 % self.num_stages
@@ -342,6 +344,7 @@ class IndexDecodeScoreKernel:
rC[None, m, n],
)
cute.arch.fence_proxy("async.shared", space="cta")
cute.arch.mbarrier_arrive(tma_empty_mbar + tma_stage)
k_start = block_id * BLOCK_K + warp_id * 32