[Minimax-M3] BF16/FP8 Indexer using MSA (#45892)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Yongye Zhu
2026-06-23 10:28:49 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Thien Tran
parent f4d5f73ffa
commit 6691f087a6
13 changed files with 1048 additions and 104 deletions
+325 -1
View File
@@ -134,6 +134,7 @@ def _reference_index_topk(
topk: int,
init_blocks: int,
local_blocks: int,
sm_scale: float = 1.0,
) -> torch.Tensor:
total_q, num_idx_heads, _ = idx_q.shape
out = torch.full(
@@ -149,7 +150,7 @@ def _reference_index_topk(
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
pages = block_table[req_id, :num_blocks]
k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1)
score = torch.einsum("qhd,kd->hqk", q.float(), k.float())
score = torch.einsum("qhd,kd->hqk", q.float(), k.float()) * sm_scale
q_pos = prefix_len + torch.arange(q_len, device=idx_q.device)
k_pos = torch.arange(k.shape[0], device=idx_q.device)
@@ -244,6 +245,270 @@ def test_prefill_index_topk_correctness():
_assert_topk_indices_equal_unordered(actual, expected)
# MSA indexer (SM100): fmha_sm100 OnlyScore for the per-block scores, then the
# Triton minimax_m3_index_topk for selection (no sparse_topk_select). Uses a
# deterministic construction (idx_q == 1, distinct e4m3-exact per-block values)
# so scores are strictly monotonic in the block id -> exact top-k agreement.
def _fmha_indexer_topk(
idx_q: torch.Tensor, # [total_q, H, 128] bf16/e4m3
index_cache: torch.Tensor, # [num_pages, 128, 128] bf16/e4m3
block_table: torch.Tensor,
q_lens: torch.Tensor,
seq_lens: torch.Tensor,
prefix_lens: torch.Tensor,
sm_scale: float,
topk: int,
) -> torch.Tensor:
"""Replicate MiniMaxM3IndexerMSAImpl's score path (single decode/prefill side)."""
from vllm.third_party.fmha_sm100.api import _fmha_sm100, _fmha_sm100_plan
num_idx_heads, head_dim = idx_q.shape[1], idx_q.shape[2]
nvp = [(s + 127) // 128 for s in seq_lens.tolist()]
kv_indices = torch.cat([block_table[r, : nvp[r]] for r in range(len(nvp))]).to(
torch.int32
)
qo = q_lens.cpu().to(torch.int32)
kv = seq_lens.cpu().to(torch.int32)
plan = _fmha_sm100_plan(
qo,
kv,
num_idx_heads,
num_kv_heads=1,
qo_offset=kv - qo,
page_size=128,
output_maxscore=True,
causal=True,
num_kv_splits=1,
)
k_pages = index_cache.view(index_cache.shape[0], 1, 128, head_dim)
_, max_score = _fmha_sm100(
idx_q,
k_pages,
k_pages,
plan,
kv_indices=kv_indices,
output_o=False,
output_maxscore=True,
sm_scale=sm_scale,
)
batch = q_lens.numel()
cu = torch.zeros(batch + 1, dtype=torch.int32, device=idx_q.device)
cu[1:] = q_lens.to(torch.int32).cumsum(0)
# max_score [H, k_tiles, total_q] -> transpose to [H, total_q, k_tiles].
return minimax_m3_index_topk(
max_score.transpose(1, 2),
cu,
prefix_lens.to(torch.int32),
int(q_lens.max()),
topk,
0, # init_blocks
0, # local_blocks
)
# e4m3-exact, strictly-increasing per-block values: with idx_q == 1 (also exact)
# the per-block scores are exact and distinct in BOTH bf16 and e4m3, so the fp8
# score path selects the same top-k as the reference (no quantization ties).
_E4M3_EXACT_VALUES = [
*range(1, 17), # 1..16 (step 1)
*range(18, 33, 2), # 18..32 (step 2)
*range(36, 65, 4), # 36..64 (step 4)
*range(72, 129, 8), # 72..128 (step 8)
]
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="fmha_sm100 indexer requires SM100 (Blackwell).",
)
@pytest.mark.parametrize("index_dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize(
("q_lens", "prefix_lens"),
[
((4, 3), (2048, 2560)), # prefill: every token sees >= 16 causal blocks
((1, 1, 1), (2048, 3000, 4096)), # decode: one query token per request
],
)
def test_fmha_sm100_indexer_matches_reference(q_lens, prefix_lens, index_dtype):
torch.manual_seed(0)
num_idx_heads, head_dim = 4, HEAD_DIM
device = "cuda"
q_lens_t = torch.tensor(q_lens, device=device, dtype=torch.int32)
prefix_lens_t = torch.tensor(prefix_lens, device=device, dtype=torch.int32)
seq_lens = prefix_lens_t + q_lens_t
batch = len(q_lens)
max_blocks = (int(seq_lens.max()) + BLOCK_SIZE - 1) // BLOCK_SIZE
assert max_blocks <= len(_E4M3_EXACT_VALUES)
num_pages = batch * max_blocks
block_table = torch.randperm(num_pages, device=device, dtype=torch.int32).reshape(
batch, max_blocks
)
idx_q = torch.ones(
int(q_lens_t.sum()), num_idx_heads, head_dim, device=device, dtype=index_dtype
)
index_cache = torch.empty(
num_pages, BLOCK_SIZE, head_dim, device=device, dtype=index_dtype
)
for r in range(batch):
for b in range(max_blocks):
index_cache[block_table[r, b]] = float(_E4M3_EXACT_VALUES[b])
sm_scale = head_dim**-0.5
actual = _fmha_indexer_topk(
idx_q,
index_cache,
block_table,
q_lens_t,
seq_lens,
prefix_lens_t,
sm_scale,
TOPK,
)
expected = _reference_index_topk(
idx_q,
index_cache,
block_table,
q_lens_t,
seq_lens,
prefix_lens_t,
TOPK,
init_blocks=0,
local_blocks=0,
sm_scale=sm_scale,
)
_assert_topk_indices_equal_unordered(actual, expected)
# Full impl-level parity: drive both MiniMaxM3IndexerMSAImpl (fmha_sm100 score +
# Triton top-k) and MiniMaxM3IndexerTritonImpl through their real metadata
# builders on the SAME CommonAttentionMetadata + index cache, and assert the
# selected blocks agree. This exercises all the metadata the impl/kernels consume
# (decode/prefill split, cu_seqlens_q rebasing, prefix_lens, kv_indices gather,
# decode_pages split) -- a metadata bug on either side shifts the causal window
# or the block->page mapping and breaks the comparison.
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="fmha_sm100 indexer requires SM100 (Blackwell).",
)
@pytest.mark.parametrize("topk", [8, 16])
def test_msa_indexer_impl_matches_triton(topk, monkeypatch):
import vllm.models.minimax_m3.common.indexer as indexer_mod
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.models.minimax_m3.common.indexer import (
MiniMaxM3IndexerTritonImpl,
MiniMaxM3IndexerTritonMetadataBuilder,
)
from vllm.models.minimax_m3.nvidia.indexer_msa import (
MiniMaxM3IndexerMSAImpl,
MiniMaxM3IndexerMSAMetadataBuilder,
)
torch.manual_seed(0)
device = torch.device("cuda")
num_idx_heads, head_dim = 4, HEAD_DIM
# TP=1: avoid requiring an initialized distributed group in a unit test.
monkeypatch.setattr(indexer_mod, "get_tensor_model_parallel_world_size", lambda: 1)
vllm_config = create_vllm_config(
block_size=BLOCK_SIZE, max_model_len=8192, max_num_batched_tokens=8192
)
vllm_config.model_config.hf_config.sparse_attention_config = {
"sparse_num_index_heads": num_idx_heads
}
# Decode-first mixed batch: 2 decode reqs (q_len 1) then 2 prefill reqs. Long
# prefixes so every token sees > TOPK causal blocks (non-trivial selection).
batch = BatchSpec(seq_lens=[2305, 2561, 2624, 2720], query_lens=[1, 1, 64, 96])
common = create_common_attn_metadata(
batch, BLOCK_SIZE, device, arange_block_indices=True
)
num_tokens = batch.compute_num_tokens()
# Deterministic index cache: distinct, monotonic per-logical-block values so
# the top-k is unambiguous (both kernels pick the same blocks, no fp ties).
block_table = common.block_table_tensor
num_pages = int(block_table.max().item()) + 1
index_cache = torch.zeros(
num_pages, BLOCK_SIZE, head_dim, device=device, dtype=DTYPE
)
for r, seq_len in enumerate(batch.seq_lens):
for b in range((seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE):
index_cache[block_table[r, b]] = float(b + 1)
index_q = torch.ones(
num_tokens, num_idx_heads * head_dim, device=device, dtype=DTYPE
)
spec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=head_dim, dtype=DTYPE
)
impl_kwargs = dict(
num_kv_heads=num_idx_heads,
scale=head_dim**-0.5,
topk_blocks=topk,
sparse_block_size=BLOCK_SIZE,
num_index_heads=num_idx_heads,
index_head_dim=head_dim,
init_blocks=0,
local_blocks=0,
)
with set_current_vllm_config(vllm_config):
msa_impl = MiniMaxM3IndexerMSAImpl(prefix="idx_msa", **impl_kwargs)
triton_impl = MiniMaxM3IndexerTritonImpl(prefix="idx_triton", **impl_kwargs)
msa_builder = MiniMaxM3IndexerMSAMetadataBuilder(
spec, [msa_impl.index_cache.prefix], vllm_config, device
)
triton_builder = MiniMaxM3IndexerTritonMetadataBuilder(
spec, [triton_impl.index_cache.prefix], vllm_config, device
)
# Both impls score against the same index keys.
msa_impl.index_cache.kv_cache = index_cache
triton_impl.index_cache.kv_cache = index_cache
# Exercise the shared persistent top-k buffer for BOTH impls: each must write
# decode ([:, :nd]) and prefill ([:, nd:]) into its buffer and return views.
# Separate buffers so the two forwards don't clobber each other.
nd = sum(q for q in batch.query_lens if q <= 1)
msa_impl.topk_indices_buffer = torch.full(
(num_idx_heads, num_tokens, topk), -2, dtype=torch.int32, device=device
)
triton_impl.topk_indices_buffer = torch.full(
(num_idx_heads, num_tokens, topk), -2, dtype=torch.int32, device=device
)
attn_metadata = {
msa_impl.index_cache.prefix: msa_builder.build(0, common),
triton_impl.index_cache.prefix: triton_builder.build(0, common),
}
with set_forward_context(attn_metadata, vllm_config):
msa_decode, msa_prefill = msa_impl(index_q)
tri_decode, tri_prefill = triton_impl(index_q)
assert msa_decode is not None and tri_decode is not None
assert msa_prefill is not None and tri_prefill is not None
_assert_topk_indices_equal_unordered(msa_decode, tri_decode)
_assert_topk_indices_equal_unordered(msa_prefill, tri_prefill)
# decode/prefill outputs are views into each impl's persistent buffer.
for impl, dec, pre in (
(msa_impl, msa_decode, msa_prefill),
(triton_impl, tri_decode, tri_prefill),
):
buf = impl.topk_indices_buffer
assert dec.data_ptr() == buf[:, :nd, :].data_ptr()
assert pre.data_ptr() == buf[:, nd:, :].data_ptr()
@pytest.mark.parametrize(
("decode_query_len", "max_decode_query_len"),
[
@@ -317,6 +582,65 @@ def test_decode_index_topk_correctness(
_assert_topk_indices_equal_unordered(actual, expected)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="fp8 e4m3 indexer cache is the SM100 (MSA) path.",
)
@pytest.mark.parametrize("num_idx_heads", [1, 4])
def test_decode_index_topk_fp8(num_idx_heads: int):
"""The fp8 (e4m3) indexer cache feeds the Triton decode kernel on the MSA
path. The kernel must score in fp32 (no scaling) so its top-k matches a
reference computed from the dequantized fp8 values."""
torch.manual_seed(0)
topk, init_blocks, local_blocks, head_dim = 8, 0, 1, 128
decode_query_len = 1
active_seq_lens = torch.tensor((129, 1025, 4097), device="cuda", dtype=torch.int32)
q_lens = torch.full_like(active_seq_lens, decode_query_len)
prefix_lens = active_seq_lens - decode_query_len
batch = active_seq_lens.numel()
max_seq_len = int(active_seq_lens.max())
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
num_pages = batch * max_blocks
block_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape(
batch, max_blocks
)
idx_q = torch.randn(
batch * decode_query_len, num_idx_heads, head_dim, device="cuda"
).to(torch.float8_e4m3fn)
index_kv_cache = torch.randn(num_pages, BLOCK_SIZE, head_dim, device="cuda").to(
torch.float8_e4m3fn
)
actual = minimax_m3_index_decode(
idx_q,
index_kv_cache,
block_table,
active_seq_lens,
max_seq_len=max_seq_len,
topk=topk,
init_blocks=init_blocks,
local_blocks=local_blocks,
num_kv_heads=num_idx_heads,
sm_scale=head_dim**-0.5,
decode_query_len=decode_query_len,
)
# Reference from the DEQUANTIZED fp8 values (the kernel computes the fp8 QK
# in fp32, so it must match an fp32 matmul of the same e4m3 values).
expected = _reference_index_topk(
idx_q.float(),
index_kv_cache.float(),
block_table,
q_lens,
active_seq_lens,
prefix_lens,
topk,
init_blocks,
local_blocks,
head_dim**-0.5,
)
_assert_topk_indices_equal_unordered(actual, expected)
# Sparse attention kernels.
def _reference_sparse_attn(
q: torch.Tensor,
@@ -278,3 +278,99 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
torch.testing.assert_close(
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
)
# ── Test 3: fp8 (e4m3) index outputs ─────────────────────────────────────────
# The fp8 score path stores index_q and the index-K cache as e4m3 while q/k/v +
# q_out stay bf16. Asserts: (1) q/k/v/q_out are bit-identical to the bf16 run
# (the index dtype must not perturb the main branch), and (2) the e4m3 index
# outputs dequantize close to the bf16 reference.
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
reason="e4m3 conversion requires CUDA SM89+.",
)
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
@pytest.mark.parametrize("block_size", [16, 64])
def test_sparse_full_fp8_index(num_tokens, block_size):
torch.manual_seed(1)
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
base, max_pos = 5_000_000.0, 4096
num_heads, num_kv_heads, num_idx_heads = 16, 4, 4
q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
iq_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
ik_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
iqsz, iksz = num_idx_heads * HEAD_DIM, HEAD_DIM
qkv0 = torch.randn(
num_tokens, qsz + 2 * kvsz + iqsz + iksz, dtype=dtype, device=device
)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
index_slot_mapping = torch.roll(slot_mapping, shifts=1)
def run(index_dtype):
qkv = qkv0.clone()
kv_cache = torch.zeros(
num_blocks,
2,
block_size,
num_kv_heads,
HEAD_DIM,
dtype=dtype,
device=device,
)
index_cache = torch.zeros(
num_blocks, block_size, HEAD_DIM, dtype=index_dtype, device=device
)
q_out = torch.empty(num_tokens, qsz, dtype=dtype, device=device)
index_q = torch.empty(num_tokens, iqsz, dtype=index_dtype, device=device)
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
q_w,
k_w,
cos_sin,
positions,
num_heads,
num_kv_heads,
ROTARY_DIM,
eps,
iq_w,
ik_w,
num_idx_heads,
slot_mapping,
index_slot_mapping,
kv_cache,
index_cache,
block_size,
q_out,
index_q,
)
return qkv, kv_cache, index_cache, q_out, index_q
qkv_bf, kvc_bf, idxc_bf, qo_bf, iq_bf = run(torch.bfloat16)
qkv_fp, kvc_fp, idxc_fp, qo_fp, iq_fp = run(torch.float8_e4m3fn)
assert iq_fp.dtype == torch.float8_e4m3fn
assert idxc_fp.dtype == torch.float8_e4m3fn
# (1) The main branch (q/k/v in qkv, q_out, kv cache) must be bit-identical:
# the index output dtype must not perturb anything else.
torch.testing.assert_close(qo_fp, qo_bf, rtol=0, atol=0)
torch.testing.assert_close(qkv_fp, qkv_bf, rtol=0, atol=0)
torch.testing.assert_close(kvc_fp, kvc_bf, rtol=0, atol=0)
# (2) Dequantized e4m3 index outputs match the bf16 reference within fp8 ulp.
torch.testing.assert_close(iq_fp.float(), iq_bf.float(), rtol=0.13, atol=0.05)
torch.testing.assert_close(idxc_fp.float(), idxc_bf.float(), rtol=0.13, atol=0.05)