[M3] Improve indexer for long-context decode (sm100) (#48582)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-07-16 18:12:15 -07:00
committed by GitHub
parent b88abb5036
commit fe784ff22e
7 changed files with 755 additions and 62 deletions
+118 -8
View File
@@ -185,6 +185,43 @@ def _assert_topk_indices_equal_unordered(
assert set(actual_row) == set(expected_row)
def _reference_decode_index_score(
idx_q: torch.Tensor,
index_kv_cache: torch.Tensor,
block_table: torch.Tensor,
seq_lens: torch.Tensor,
decode_query_len: int,
score_block_stride: int,
) -> torch.Tensor:
total_q, num_idx_heads, _ = idx_q.shape
out = torch.full(
(num_idx_heads, total_q, score_block_stride),
-float("inf"),
device=idx_q.device,
dtype=torch.float32,
)
for req_id, seq_len in enumerate(seq_lens.tolist()):
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
token_start = req_id * decode_query_len
q = idx_q[token_start : token_start + decode_query_len].float()
pages = block_table[req_id, :num_blocks]
k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1).float()
score = torch.einsum("qhd,kd->hqk", q, k)
q_pos = (
seq_len
- decode_query_len
+ torch.arange(decode_query_len, device=idx_q.device)
)
k_pos = torch.arange(k.shape[0], device=idx_q.device)
score.masked_fill_(k_pos[None, :] > q_pos[:, None], -float("inf"))
out[:, token_start : token_start + decode_query_len, :num_blocks] = (
score.reshape(num_idx_heads, decode_query_len, num_blocks, BLOCK_SIZE)
.max(dim=3)
.values
)
return out
def test_prefill_index_topk_correctness():
topk = 6
init_blocks = 0
@@ -383,8 +420,8 @@ def test_fmha_sm100_indexer_matches_reference(q_lens, prefix_lens, index_dtype):
_assert_topk_indices_equal_unordered(actual, expected)
# Full impl-level parity: drive both MiniMaxM3IndexerMSAImpl (fmha_sm100 score +
# Triton top-k) and MiniMaxM3IndexerTritonImpl through their real metadata
# Full impl-level parity: drive both MiniMaxM3IndexerMSAImpl (fmha/CuteDSL score
# + unified top-k) and MiniMaxM3IndexerTritonImpl through their real metadata
# builders on the SAME CommonAttentionMetadata + index cache, and assert the
# selected blocks agree. This exercises all the metadata the impl/kernels consume
# (decode/prefill split, cu_seqlens_q rebasing, prefix_lens, kv_indices gather,
@@ -395,7 +432,8 @@ def test_fmha_sm100_indexer_matches_reference(q_lens, prefix_lens, index_dtype):
reason="fmha_sm100 indexer requires SM100 (Blackwell).",
)
@pytest.mark.parametrize("topk", [8, 16])
def test_msa_indexer_impl_matches_triton(topk, monkeypatch):
@pytest.mark.parametrize("index_dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_msa_indexer_impl_matches_triton(topk, index_dtype, monkeypatch):
import vllm.models.minimax_m3.common.indexer as indexer_mod
from tests.v1.attention.utils import (
BatchSpec,
@@ -439,13 +477,13 @@ def test_msa_indexer_impl_matches_triton(topk, monkeypatch):
block_table = common.block_table_tensor
num_pages = int(block_table.max().item()) + 1
index_cache = torch.zeros(
num_pages, BLOCK_SIZE, head_dim, device=device, dtype=DTYPE
num_pages, BLOCK_SIZE, head_dim, device=device, dtype=index_dtype
)
for r, seq_len in enumerate(batch.seq_lens):
for b in range((seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE):
index_cache[block_table[r, b]] = float(b + 1)
index_q = torch.ones(
num_tokens, num_idx_heads * head_dim, device=device, dtype=DTYPE
num_tokens, num_idx_heads * head_dim, device=device, dtype=index_dtype
)
spec = MLAAttentionSpec(
@@ -588,9 +626,8 @@ def test_decode_index_topk_correctness(
)
@pytest.mark.parametrize("num_idx_heads", [1, 4])
def test_decode_index_topk_fp8(num_idx_heads: int):
"""The fp8 (e4m3) indexer cache feeds the Triton decode kernel on the MSA
path. The kernel must score in fp32 (no scaling) so its top-k matches a
reference computed from the dequantized fp8 values."""
"""The standalone Triton path must score FP8 inputs in FP32 so its top-k
matches a reference computed from the dequantized FP8 values."""
torch.manual_seed(0)
topk, init_blocks, local_blocks, head_dim = 8, 0, 1, 128
decode_query_len = 1
@@ -641,6 +678,79 @@ def test_decode_index_topk_fp8(num_idx_heads: int):
_assert_topk_indices_equal_unordered(actual, expected)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="CuteDSL index decode score requires Blackwell.",
)
@pytest.mark.parametrize(
("dtype", "decode_query_len", "max_decode_query_len"),
[
(torch.bfloat16, 1, 1),
(torch.bfloat16, 3, 8),
(torch.float8_e4m3fn, 1, 1),
(torch.float8_e4m3fn, 3, 8),
(torch.float8_e4m3fn, 8, 8),
],
)
def test_decode_index_score_cutedsl_correctness(
dtype: torch.dtype,
decode_query_len: int,
max_decode_query_len: int,
):
pytest.importorskip("cutlass")
from vllm.models.minimax_m3.nvidia.ops import (
minimax_m3_index_decode_score_cutedsl,
)
torch.manual_seed(0)
init_blocks, local_blocks = 0, 0
num_idx_heads, head_dim = 4, 128
active_seq_lens = torch.tensor((1025, 4097), device="cuda", dtype=torch.int32)
batch = active_seq_lens.numel()
total_q = batch * decode_query_len
max_seq_len = int(active_seq_lens.max())
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
score_block_stride = ((max_blocks + 15) // 16) * 16
num_pages = batch * max_blocks
block_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape(
batch, max_blocks
)
idx_q = torch.randn(total_q, num_idx_heads, head_dim, device="cuda").to(dtype)
index_kv_cache = torch.randn(num_pages, BLOCK_SIZE, head_dim, device="cuda").to(
dtype
)
unified_score = torch.full(
(total_q, num_idx_heads, score_block_stride),
-float("inf"),
device="cuda",
dtype=torch.float32,
)
score = unified_score.transpose(0, 1)
minimax_m3_index_decode_score_cutedsl(
idx_q,
index_kv_cache,
block_table,
active_seq_lens,
max_seq_len=max_seq_len,
init_blocks=init_blocks,
local_blocks=local_blocks,
num_kv_heads=num_idx_heads,
decode_query_len=decode_query_len,
max_decode_query_len=max_decode_query_len,
score_out=score,
)
expected = _reference_decode_index_score(
idx_q,
index_kv_cache,
block_table,
active_seq_lens,
decode_query_len,
score_block_stride,
)
torch.testing.assert_close(score, expected)
# Sparse attention kernels.
def _reference_sparse_attn(
q: torch.Tensor,