[Perf][MiniMax-M3] MSA prefill attends with the E4M3 query; bump fmha_sm100 to 968061a (#60684)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-11 13:40:45 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f8471164bf
commit 2eb491e18b
4 changed files with 176 additions and 32 deletions
@@ -782,6 +782,46 @@ def _make_nvfp4_kv_cache(
return kv_cache, torch.cat(dequantized, dim=-1).to(torch.bfloat16)
def _e4m3_staged_kv_reference(
kv_reference: torch.Tensor, k_scale: torch.Tensor, v_scale: torch.Tensor
) -> torch.Tensor:
"""The dequantized NVFP4 cache as an E4M3-query kernel reads it: each
code * block_scale rounded to E4M3, then the global scale."""
sides = []
for side, global_scale in zip(
kv_reference.float().split(HEAD_DIM, dim=-1), (k_scale, v_scale)
):
sides.append(
(side / global_scale).to(torch.float8_e4m3fn).float() * global_scale
)
return torch.cat(sides, dim=-1).to(torch.bfloat16)
def _assert_close_fp8_prefill(actual: torch.Tensor, expected: torch.Tensor) -> None:
"""The fp8-q prefill rounds each split's probabilities to E4M3, and the splits
follow the SM count, so a rare element moves past the BF16 tolerance."""
close = torch.isclose(actual, expected, atol=0.05, rtol=0.05)
assert close.float().mean().item() > 1 - 1e-4
torch.testing.assert_close(actual, expected, atol=0.25, rtol=0.05)
def _record_query_dtypes(
monkeypatch: pytest.MonkeyPatch, kernel_name: str
) -> list[torch.dtype]:
"""Record the query dtype of every ``fmha_sm100.sparse.<kernel_name>`` call."""
from vllm.third_party.fmha_sm100 import sparse as msa_sparse
kernel = getattr(msa_sparse, kernel_name)
query_dtypes: list[torch.dtype] = []
def record(q, *args, **kwargs):
query_dtypes.append(q.dtype)
return kernel(q, *args, **kwargs)
monkeypatch.setattr(msa_sparse, kernel_name, record)
return query_dtypes
def test_nvfp4_kv_cache_views_address_head_slots() -> None:
"""Slot 2h (K) and 2h + 1 (V) hold head h's data then scales, so each head
is one contiguous run of the page: a TP rank's page is a byte slice."""
@@ -898,27 +938,14 @@ def test_msa_cutlass_decode_nvfp4_matches_triton_on_dequantized_cache(
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads"),
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
)
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
num_q_heads: int,
def _make_prefill_metadata(
q_lens: list[int],
seq_lens_list: list[int],
block_table: torch.Tensor,
num_kv_heads: int,
) -> None:
"""MSA prefill reads NVFP4 pages as their exact dequantized values,
including chunked prefill over cached context, and takes q from its fp8
copy (the only one the fused insert writes on this path)."""
torch.manual_seed(0)
q_lens = [300, 128, 77]
seq_lens_list = [300, 640, 1100]
block_table, num_pages = _make_paged_layout(seq_lens_list)
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
kv_cache, kv_reference = _make_nvfp4_kv_cache(
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
)
) -> tuple[MiniMaxM3SparseMetadata, torch.Tensor]:
"""Prefill-only metadata (chunks over cached context) and a top-k selecting
every causally visible page (<= TOPK per query), ending with the local one."""
total_q = sum(q_lens)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
context_lens = seq_lens - torch.tensor(q_lens, dtype=torch.int32, device="cuda")
@@ -948,7 +975,6 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
prefill=prefill,
)
# Every causally visible page (<= TOPK per query), ending with the local one.
topk = torch.full(
(total_q, num_kv_heads, TOPK), -1, dtype=torch.int32, device="cuda"
)
@@ -959,6 +985,35 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
assert visible <= TOPK
topk[token, :, :visible] = torch.arange(visible, device="cuda")
token += 1
return metadata, topk
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads"),
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
)
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
monkeypatch: pytest.MonkeyPatch,
num_q_heads: int,
num_kv_heads: int,
) -> None:
"""MSA prefill reads NVFP4 pages as their dequantized values, staged to E4M3
for the E4M3 q, including chunked prefill over cached context; it attends
with the fp8 q (the only one the fused insert writes on this path)."""
torch.manual_seed(0)
q_lens = [300, 128, 77]
seq_lens_list = [300, 640, 1100]
block_table, num_pages = _make_paged_layout(seq_lens_list)
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
kv_cache, kv_reference = _make_nvfp4_kv_cache(
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
)
metadata, topk = _make_prefill_metadata(
q_lens, seq_lens_list, block_table, num_kv_heads
)
total_q = sum(q_lens)
layer_name = "model.layers.0.self_attn.attn"
q_scale = 0.5
@@ -976,6 +1031,7 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
/ q_scale
).to(torch.float8_e4m3fn)
query = query_fp8.to(torch.bfloat16) * q_scale
kv_reference = _e4m3_staged_kv_reference(kv_reference, k_scale, v_scale)
common = dict(
num_heads=num_q_heads,
head_size=HEAD_DIM,
@@ -984,6 +1040,7 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
)
query_dtypes = _record_query_dtypes(monkeypatch, "sparse_atten_nvfp4_kv_func")
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={layer_name: metadata},
@@ -1001,7 +1058,8 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
query_fp8=query_fp8,
)
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
assert query_dtypes == [torch.float8_e4m3fn]
_assert_close_fp8_prefill(actual, expected)
def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
@@ -1079,3 +1137,80 @@ def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
)
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads"),
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
)
def test_msa_prefill_reads_query_fp8_over_fp8_cache(
monkeypatch: pytest.MonkeyPatch,
num_q_heads: int,
num_kv_heads: int,
) -> None:
"""With CUTLASS decode on, FP8 prefill (incl. chunks over cached context)
attends with the fp8 q the fused insert writes, not a BF16 copy, and
matches Triton on the dequantized q."""
torch.manual_seed(0)
q_lens = [300, 128, 77]
seq_lens_list = [300, 640, 1100]
block_table, num_pages = _make_paged_layout(seq_lens_list)
kv_cache = (
torch.randn(
num_pages,
num_kv_heads,
BLOCK_SIZE,
2 * HEAD_DIM,
dtype=torch.bfloat16,
device="cuda",
)
* 0.5
).to(torch.float8_e4m3fn)
metadata, topk = _make_prefill_metadata(
q_lens, seq_lens_list, block_table, num_kv_heads
)
layer_name = "model.layers.0.self_attn.attn"
q_scale = 0.5
layer = SimpleNamespace(
layer_name=layer_name,
topk_indices_buffer=topk,
_q_scale_float=q_scale,
)
query_fp8 = (
torch.randn(
sum(q_lens), num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
/ q_scale
).to(torch.float8_e4m3fn)
query = query_fp8.to(torch.bfloat16) * q_scale
common = dict(
num_heads=num_q_heads,
head_size=HEAD_DIM,
scale=SM_SCALE,
num_kv_heads=num_kv_heads,
kv_cache_dtype="fp8",
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
)
impl = MiniMaxM3SparseMSAImpl(**common, msa_decode_backend="cutlass")
assert impl.use_cutlass_decode
query_dtypes = _record_query_dtypes(monkeypatch, "sparse_atten_func")
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={layer_name: metadata},
slot_mapping={},
)
with override_forward_context(forward_context):
expected = MiniMaxM3SparseTritonImpl(**common).forward(
layer, query, kv_cache, torch.empty_like(query)
)
actual = impl.forward(
layer,
torch.full_like(query, float("nan")),
kv_cache,
torch.zeros_like(query),
query_fp8=query_fp8,
)
current_platform.synchronize()
assert query_dtypes == [torch.float8_e4m3fn]
_assert_close_fp8_prefill(actual, expected)