[Attention][MiniMax-M3] NVFP4 KV cache on the MSA sparse attention path (#59300)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-01 14:39:55 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent ea84051819
commit 7566d83bd3
12 changed files with 1297 additions and 73 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for MiniMax M3 CUTLASS sparse decode."""
"""Correctness tests for MiniMax M3 MSA sparse attention (CUTLASS decode and
NVFP4 KV cache)."""
import math
from types import SimpleNamespace
@@ -8,6 +9,7 @@ from types import SimpleNamespace
import pytest
import torch
from tests.kernels.quantization.nvfp4_utils import dequant_nvfp4_kv_cache
from vllm import _custom_ops as ops
from vllm.config import AttentionConfig, CUDAGraphMode
from vllm.forward_context import ForwardContext, override_forward_context
@@ -18,6 +20,7 @@ from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseBackend,
MiniMaxM3SparseMetadata,
MiniMaxM3SparseMetadataBuilder,
MiniMaxM3SparsePrefillMetadata,
MiniMaxM3SparseTritonImpl,
select_main_backend_and_impl_cls,
)
@@ -28,6 +31,7 @@ from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
MSACutlassDecodePlanCache,
msa_cutlass_sparse_decode,
nvfp4_kv_cache_views,
prepare_decode_metadata,
should_prepare_decode_metadata,
)
@@ -36,6 +40,7 @@ from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
MiniMaxM3SparseMSADecodeMetadata,
MiniMaxM3SparseMSAImpl,
MiniMaxM3SparseMSAMetadataBuilder,
MiniMaxM3SparseMSANvfp4Backend,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -164,6 +169,53 @@ def test_msa_cutlass_decode_static_dispatch_requires_sm100(
)
@pytest.mark.parametrize("decode_backend", ["cutlass", "triton"])
def test_msa_cutlass_decode_static_dispatch_nvfp4_has_no_fallback(
decode_backend: str,
) -> None:
"""NVFP4 has no Triton decode, so it plans CUTLASS for any batch size."""
assert should_prepare_decode_metadata(
1,
DEFAULT_QUERY_LEN,
decode_backend=decode_backend, # type: ignore[arg-type]
num_q_heads=16,
num_kv_heads=1,
kv_cache_dtype="nvfp4",
page_size=BLOCK_SIZE,
topk_blocks=TOPK,
)
def test_nvfp4_selects_msa_nvfp4_backend_and_requires_sm100(
monkeypatch: pytest.MonkeyPatch,
) -> None:
backend_cls, impl_cls = select_main_backend_and_impl_cls(
topk_blocks=TOPK,
kv_cache_dtype="nvfp4",
num_kv_heads=1,
)
assert backend_cls is MiniMaxM3SparseMSANvfp4Backend
assert impl_cls is MiniMaxM3SparseMSAImpl
with pytest.raises(ValueError, match="nvfp4_4over6"):
select_main_backend_and_impl_cls(
topk_blocks=TOPK,
kv_cache_dtype="nvfp4_4over6",
num_kv_heads=1,
)
monkeypatch.setattr(
current_platform,
"is_device_capability_family",
lambda _: False,
)
with pytest.raises(ValueError, match="NVFP4"):
select_main_backend_and_impl_cls(
topk_blocks=TOPK,
kv_cache_dtype="nvfp4",
num_kv_heads=1,
)
@pytest.mark.parametrize(
("backend", "expected"),
[
@@ -639,3 +691,391 @@ def test_msa_cutlass_decode_matches_triton_with_interleaved_cache(
graph.replay()
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
def _make_paged_layout(
seq_lens_list: list[int],
) -> tuple[torch.Tensor, int]:
"""Block table over a shuffled physical page pool."""
pages_per_request = [math.ceil(seq_len / BLOCK_SIZE) for seq_len in seq_lens_list]
num_pages = sum(pages_per_request)
block_table = torch.zeros(
len(seq_lens_list), max(pages_per_request), dtype=torch.int32, device="cuda"
)
physical_pages = torch.randperm(num_pages, dtype=torch.int32, device="cuda")
offset = 0
for request, request_pages in enumerate(pages_per_request):
block_table[request, :request_pages] = physical_pages[
offset : offset + request_pages
]
offset += request_pages
return block_table, num_pages
def _make_nvfp4_kv_cache(
num_pages: int,
num_kv_heads: int,
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Write random K/V through the NVFP4 ``reshape_and_cache_flash`` writer,
whose pages hold every head's data then every head's scales per side, and
repack them into head slots.
Returns the packed HND cache ``[pages, 2 * Hkv, 128, 72]`` and its BF16
dequantization ``[pages, Hkv, 128, 2 * 128]`` for the Triton reference.
"""
num_slots = num_pages * BLOCK_SIZE
key = torch.randn(
num_slots, num_kv_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
value = torch.randn_like(key)
written = torch.zeros(
num_pages,
2 * num_kv_heads,
BLOCK_SIZE,
HEAD_DIM // 2 + HEAD_DIM // 16,
dtype=torch.uint8,
device="cuda",
)
k_cache, v_cache = written.transpose(1, 2).split(num_kv_heads, dim=-2)
ops.reshape_and_cache_flash(
key,
value,
k_cache,
v_cache,
torch.arange(num_slots, dtype=torch.int64, device="cuda"),
kv_cache_dtype,
k_scale,
v_scale,
)
kv_cache = torch.empty_like(written)
head_slot_views = iter(nvfp4_kv_cache_views(kv_cache))
dequantized = []
for side, global_scale, swizzled in (
(written[:, :num_kv_heads], k_scale, False),
(written[:, num_kv_heads:], v_scale, True),
):
data = side.as_strided(
(num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2),
(side.stride(0), BLOCK_SIZE * HEAD_DIM // 2, HEAD_DIM // 2, 1),
)
scale = side.as_strided(
(num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 16),
(side.stride(0), BLOCK_SIZE * HEAD_DIM // 16, HEAD_DIM // 16, 1),
side.storage_offset() + num_kv_heads * BLOCK_SIZE * HEAD_DIM // 2,
)
next(head_slot_views).copy_(data)
next(head_slot_views).copy_(scale)
dequantized.append(
dequant_nvfp4_kv_cache(
data,
scale,
global_scale.item(),
HEAD_DIM,
BLOCK_SIZE,
swizzled_scales=swizzled,
)
)
return kv_cache, torch.cat(dequantized, dim=-1).to(torch.bfloat16)
def test_nvfp4_kv_cache_views_address_head_slots() -> None:
"""Slot 2h (K) and 2h + 1 (V) hold head h's data then scales, so each head
is one contiguous run of the page: a TP rank's page is a byte slice."""
num_kv_heads, slot_bytes = 4, BLOCK_SIZE * (HEAD_DIM // 2 + HEAD_DIM // 16)
kv_cache = torch.randint(
0,
256,
(3, 2 * num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2 + HEAD_DIM // 16),
dtype=torch.uint8,
)
k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache)
runs = kv_cache.view(3, num_kv_heads, 2 * slot_bytes)
for head in range(num_kv_heads):
expected = torch.cat(
[v[:, head].flatten(1) for v in (k_data, k_sf, v_data, v_sf)], dim=1
)
assert torch.equal(runs[:, head], expected)
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads", "num_request_pairs", "query_len", "capture"),
[
pytest.param(64, 4, 1, 1, True, id="tp1-small-batch-query-len-1"),
pytest.param(64, 4, 8, 8, False, id="tp1-query-len-8"),
pytest.param(16, 1, 16, 4, True, id="tp4-query-len-4"),
],
)
def test_msa_cutlass_decode_nvfp4_matches_triton_on_dequantized_cache(
num_q_heads: int,
num_kv_heads: int,
num_request_pairs: int,
query_len: int,
capture: bool,
) -> None:
"""CUTLASS decode reads vLLM's packed NVFP4 pages, including swizzled V
scales and device global scales, as their exact dequantized values."""
torch.manual_seed(0)
seq_lens_list = [257, 513] * num_request_pairs
seq_lens_cpu = torch.tensor(seq_lens_list, dtype=torch.int32)
seq_lens = seq_lens_cpu.cuda()
block_table, num_pages = _make_paged_layout(seq_lens_list)
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
kv_cache, kv_reference = _make_nvfp4_kv_cache(
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
)
num_query_tokens = len(seq_lens_list) * query_len
query = torch.randn(
num_query_tokens, num_q_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
query_fp8 = query.to(torch.float8_e4m3fn)
topk = _make_topk(seq_lens_list, num_kv_heads, query_len)
expected = torch.empty_like(query)
minimax_m3_sparse_attn_decode(
query_fp8.to(torch.bfloat16),
kv_reference,
topk.transpose(0, 1),
block_table,
seq_lens,
num_kv_heads,
SM_SCALE,
expected,
query_len,
)
assert should_prepare_decode_metadata(
len(seq_lens_list),
query_len,
decode_backend="triton",
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
kv_cache_dtype="nvfp4",
page_size=BLOCK_SIZE,
topk_blocks=TOPK,
)
metadata = prepare_decode_metadata(
block_table,
seq_lens,
seq_lens_cpu,
query_len,
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
page_size=BLOCK_SIZE,
topk_blocks=TOPK,
)
actual = torch.zeros_like(query)
def run() -> None:
msa_cutlass_sparse_decode(
query_fp8,
kv_cache,
topk,
actual,
metadata,
scale=SM_SCALE,
q_scale_float=1.0,
k_scale_float=1.0,
v_scale_float=1.0,
k_scale=k_scale,
v_scale=v_scale,
)
if capture:
run()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
actual.zero_()
graph.replay()
else:
run()
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads"),
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
)
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
num_q_heads: int,
num_kv_heads: int,
) -> None:
"""MSA prefill reads NVFP4 pages as their exact dequantized values,
including chunked prefill over cached context, and takes q from its fp8
copy (the only one the fused insert writes on this path)."""
torch.manual_seed(0)
q_lens = [300, 128, 77]
seq_lens_list = [300, 640, 1100]
block_table, num_pages = _make_paged_layout(seq_lens_list)
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
kv_cache, kv_reference = _make_nvfp4_kv_cache(
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
)
total_q = sum(q_lens)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
context_lens = seq_lens - torch.tensor(q_lens, dtype=torch.int32, device="cuda")
cu_seqlens_q = torch.zeros(len(q_lens) + 1, dtype=torch.int32, device="cuda")
cu_seqlens_q[1:] = torch.tensor(q_lens, device="cuda").cumsum(0)
cu_seqlens_k = torch.zeros_like(cu_seqlens_q)
cu_seqlens_k[1:] = seq_lens.cumsum(0)
prefill = MiniMaxM3SparsePrefillMetadata(
cu_seqlens_q=cu_seqlens_q,
cu_seqlens_k=cu_seqlens_k,
seq_lens=seq_lens,
context_lens=context_lens,
block_table=block_table,
max_query_len=max(q_lens),
max_seq_len=max(seq_lens_list),
total_kv_blocks=sum(math.ceil(n / BLOCK_SIZE) for n in seq_lens_list),
)
metadata = MiniMaxM3SparseMetadata(
seq_lens=seq_lens,
max_seq_len=max(seq_lens_list),
slot_mapping=torch.empty(total_q, dtype=torch.int64, device="cuda"),
num_actual_tokens=total_q,
num_decodes=0,
num_decode_tokens=0,
num_prefills=len(q_lens),
num_prefill_tokens=total_q,
prefill=prefill,
)
# Every causally visible page (<= TOPK per query), ending with the local one.
topk = torch.full(
(total_q, num_kv_heads, TOPK), -1, dtype=torch.int32, device="cuda"
)
token = 0
for q_len, seq_len in zip(q_lens, seq_lens_list):
for local_q in range(q_len):
visible = (seq_len - q_len + local_q) // BLOCK_SIZE + 1
assert visible <= TOPK
topk[token, :, :visible] = torch.arange(visible, device="cuda")
token += 1
layer_name = "model.layers.0.self_attn.attn"
q_scale = 0.5
layer = SimpleNamespace(
layer_name=layer_name,
topk_indices_buffer=topk,
_q_scale_float=q_scale,
_k_scale=k_scale,
_v_scale=v_scale,
)
query_fp8 = (
torch.randn(
total_q, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
/ q_scale
).to(torch.float8_e4m3fn)
query = query_fp8.to(torch.bfloat16) * q_scale
common = dict(
num_heads=num_q_heads,
head_size=HEAD_DIM,
scale=SM_SCALE,
num_kv_heads=num_kv_heads,
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
)
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={layer_name: metadata},
slot_mapping={},
)
with override_forward_context(forward_context):
expected = MiniMaxM3SparseTritonImpl(**common).forward(
layer, query, kv_reference, torch.empty_like(query)
)
actual = MiniMaxM3SparseMSAImpl(**common, kv_cache_dtype="nvfp4").forward(
layer,
torch.full_like(query, float("nan")),
kv_cache,
torch.zeros_like(query),
query_fp8=query_fp8,
)
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
"""With CUTLASS decode on but not planned for the batch, the Triton decode
fallback attends with the dequantized fp8 q, the only q written."""
torch.manual_seed(0)
num_q_heads, num_kv_heads, q_scale = 16, 1, 0.5
seq_lens_list = [257, 513, 129]
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
block_table, num_pages = _make_paged_layout(seq_lens_list)
kv_cache = (
torch.randn(
num_pages,
num_kv_heads,
BLOCK_SIZE,
2 * HEAD_DIM,
dtype=torch.bfloat16,
device="cuda",
)
* 0.25
).to(torch.float8_e4m3fn)
num_tokens = len(seq_lens_list)
metadata = MiniMaxM3SparseMetadata(
seq_lens=seq_lens,
max_seq_len=max(seq_lens_list),
slot_mapping=torch.empty(num_tokens, dtype=torch.int64, device="cuda"),
num_actual_tokens=num_tokens,
num_decodes=num_tokens,
num_decode_tokens=num_tokens,
num_prefills=0,
num_prefill_tokens=0,
decode=MiniMaxM3SparseMSADecodeMetadata(
seq_lens=seq_lens,
block_table=block_table,
decode_query_len=1,
msa_cutlass=None,
),
)
layer_name = "model.layers.0.self_attn.attn"
layer = SimpleNamespace(
layer_name=layer_name,
topk_indices_buffer=_make_topk(seq_lens_list, num_kv_heads, 1),
_q_scale_float=q_scale,
)
query_fp8 = torch.randn(
num_tokens, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
).to(torch.float8_e4m3fn)
query = query_fp8.to(torch.bfloat16) * q_scale
common = dict(
num_heads=num_q_heads,
head_size=HEAD_DIM,
scale=SM_SCALE,
num_kv_heads=num_kv_heads,
kv_cache_dtype="fp8",
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
)
impl = MiniMaxM3SparseMSAImpl(**common, msa_decode_backend="cutlass")
assert impl.use_cutlass_decode
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={layer_name: metadata},
slot_mapping={},
)
with override_forward_context(forward_context):
expected = MiniMaxM3SparseTritonImpl(**common).forward(
layer, query, kv_cache, torch.empty_like(query)
)
actual = impl.forward(
layer,
torch.full_like(query, float("nan")),
kv_cache,
torch.zeros_like(query),
query_fp8=query_fp8,
)
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
@@ -320,6 +320,38 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
)
# Without q_out, q is emitted only in fp8: the same bytes, while qkv keeps
# its raw q and every other output matches the run above.
outputs = [qkv_orig.clone(), torch.empty_like(q_fp8), torch.empty_like(index_q)]
caches = [torch.zeros_like(kv_cache), torch.zeros_like(index_cache)]
ops.fused_minimax_m3_qknorm_rope_kv_insert(
outputs[0],
q_w,
k_w,
cos_sin,
positions,
num_heads,
num_kv_heads,
ROTARY_DIM,
eps,
iq_w,
ik_w,
num_idx_heads,
slot_mapping,
index_slot_mapping,
*caches,
block_size,
None,
outputs[2],
kv_cache_dtype,
q_fp8_out=outputs[1],
q_fp8_scale=0.5,
)
for actual, expected in zip(
outputs + caches, [qkv, q_fp8, index_q, kv_cache, index_cache]
):
assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8))
@pytest.mark.parametrize("num_tokens", [1, 64, 513])
@pytest.mark.parametrize("block_size", [16, 64])
@@ -441,8 +473,8 @@ def test_sparse_skip_index_branch(num_tokens, block_size, kv_cache_dtype):
# ── Test 4: fp8 (e4m3) index outputs ─────────────────────────────────────────
# The fp8 score path stores index_q and the index-K cache as e4m3 while q/k/v +
# q_out stay bf16. Asserts: (1) q/k/v/q_out are bit-identical to the bf16 run
# (the index dtype must not perturb the main branch), and (2) the e4m3 index
# outputs dequantize close to the bf16 reference.
# (the index dtype must not perturb the main branch), and (2) index_q is the
# e4m3 cast of the bf16 run's index_q, and the index-K cache is within fp8 ulp.
@pytest.mark.skipif(
@@ -528,6 +560,196 @@ def test_sparse_full_fp8_index(num_tokens, block_size):
torch.testing.assert_close(qkv_fp, qkv_bf, rtol=0, atol=0)
torch.testing.assert_close(kvc_fp, kvc_bf, rtol=0, atol=0)
# (2) Dequantized e4m3 index outputs match the bf16 reference within fp8 ulp.
torch.testing.assert_close(iq_fp.float(), iq_bf.float(), rtol=0.13, atol=0.05)
# (2) index_q is e4m3(bf16(q)); the index-K cache is within fp8 ulp.
assert torch.equal(
iq_fp.view(torch.uint8), iq_bf.to(torch.float8_e4m3fn).view(torch.uint8)
)
torch.testing.assert_close(idxc_fp.float(), idxc_bf.float(), rtol=0.13, atol=0.05)
# ── Test 5: sparse mode with an NVFP4 main KV cache ──────────────────────────
# The NVFP4 inserts quantize x * sf_scale, where sf_scale = 1 / the global
# scale: per 16-value block,
# scale = min(448, max(E4M3(max(amax, 1e-12) / 6), 1/512)) and
# code = sign | RNE E2M1 of min(|x / scale|, 6), with zero stored as +0.
E2M1_MIDPOINTS = (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)
def _div(a, b):
# Division by a Python scalar would become a reciprocal multiply.
return a / torch.as_tensor(b, dtype=a.dtype, device=a.device)
def nvfp4_ref(x, sf_scale):
"""Returns the E2M1 codes [..., 128] and E4M3 scale bytes [..., 8] of x."""
blocks = x.float().unflatten(-1, (-1, 16))
amax = blocks.abs().amax(-1, keepdim=True).clamp_min(1e-12)
scale = (sf_scale * _div(amax, 6.0)).clamp(max=448.0)
scale = scale.to(torch.float8_e4m3fn).float().clamp(min=2.0**-9)
q = _div(blocks * sf_scale, scale)
# Alternating > / >= rounds exact midpoints to the even code.
mag = sum(
(q.abs() >= m) if i % 2 else (q.abs() > m) for i, m in enumerate(E2M1_MIDPOINTS)
).to(torch.uint8)
codes = torch.where(mag > 0, mag | (torch.signbit(q).to(torch.uint8) << 3), 0)
scale_bytes = scale.squeeze(-1).to(torch.float8_e4m3fn).view(torch.uint8)
return codes.flatten(-2), scale_bytes
def nvfp4_pages(sides, slot_mapping, num_blocks, block_size):
"""Expected NVFP4 pages [nb, len(sides) * nkv, bs, 72] for the K (then V)
side given as (x [N, nkv, 128], sf_scale). Slot h * len(sides) + side is
head h's packed E2M1 data [bs, 64] (low nibble = even element), then its
E4M3 scales [bs, 8]: linear for K, 4x4 token-quad swizzled for V."""
nkv, ns, device = sides[0][0].shape[1], HEAD_DIM // 16, slot_mapping.device
row_bytes, data_bytes = HEAD_DIM // 2 + ns, block_size * HEAD_DIM // 2
pages = torch.zeros(
num_blocks,
nkv,
len(sides),
block_size * row_bytes,
dtype=torch.uint8,
device=device,
)
valid = slot_mapping >= 0
blk, tok = slot_mapping[valid] // block_size, slot_mapping[valid] % block_size
g, t = torch.arange(ns, device=device), tok[:, None]
scale_offsets = (t * ns + g, ((t // 4) * 4 + g // 2) * ns + (g % 2) * 4 + t % 4)
for side, (x, sf_scale) in enumerate(sides):
codes, scales = nvfp4_ref(x[valid], sf_scale)
data = pages[:, :, side, :data_bytes].view(num_blocks, nkv, block_size, -1)
data[blk, :, tok] = codes[..., 0::2] | (codes[..., 1::2] << 4)
scale_region = pages[:, :, side, data_bytes:]
for h in range(nkv):
scale_region[blk[:, None], h, scale_offsets[side]] = scales[:, h]
return pages.view(num_blocks, -1, block_size, row_bytes)
def nvfp4_edge_values(num_rows, global_scale):
"""[num_rows, 128] bf16 whose 16-value blocks, divided by global_scale, hit
the quantizer's corner cases: exact E2M1 midpoints under non-power-of-2 scales
(blocks 0-1), amax under the 1/512 scale floor (2), all -0 (3), and
saturation with |x| > 4096 up to bf16 max (4) or with inf (5)."""
g = torch.Generator().manual_seed(0)
x = torch.randn(num_rows, 8, 16, generator=g)
sign = torch.where(torch.rand(x.shape, generator=g) < 0.5, -1.0, 1.0)
steps = torch.tensor(E2M1_MIDPOINTS + (0.5, 1.0, 3.0))
mant = 1 + torch.randint(1, 8, (num_rows, 2, 1), generator=g) / 8
scale = mant * 2.0 ** torch.randint(-6, 3, (num_rows, 2, 1), generator=g)
x[:, :2] = steps[torch.randint(len(steps), (num_rows, 2, 16), generator=g)] * scale
x[:, :2, :1] = 6 * scale # amax = 6 * scale, so the block scale is `scale`
x[:, 2] = torch.rand(num_rows, 16, generator=g) * 2.0**-8
x[:, 4] *= 5000
x = x * sign * global_scale
x[:, 3] = -0.0
x[:, 4, 0] = sign[:, 4, 0] * torch.finfo(torch.bfloat16).max
x[:, 5, 0] = sign[:, 5, 0] * float("inf")
return x.flatten(1).to(torch.bfloat16)
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
reason="NVFP4 KV caches need SM100",
)
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
@pytest.mark.parametrize("num_kv_heads", [1, 4])
@pytest.mark.parametrize("block_size", [16, 128])
@pytest.mark.parametrize("kv_scales", [(0.5, 2.0), (1.0, 1.0)])
def test_sparse_full_nvfp4(num_tokens, num_kv_heads, block_size, kv_scales):
"""The fused NVFP4 insert writes the reference quantizer's bytes (packed
E2M1, linear K / swizzled V E4M3 block scales) for the kernel's own
post-RoPE k and for a raw v that hits the quantizer's corner cases, and
skips padded slots. Unit global scales take the kernel's separate fast
path."""
kv_cache_dtype = "nvfp4"
torch.manual_seed(5)
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
base, max_pos = 5_000_000.0, 4096
num_heads, num_idx_heads = 4 * num_kv_heads, 1
weights = [
torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1 for _ in range(4)
]
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
iqsz = num_idx_heads * HEAD_DIM
qkv = torch.randn(
num_tokens, qsz + 2 * kvsz + iqsz + HEAD_DIM, dtype=dtype, device=device
)
k_scale, v_scale = (
torch.tensor(s, dtype=torch.float32, device=device) for s in kv_scales
)
qkv[:, qsz + kvsz : qsz + 2 * kvsz] = nvfp4_edge_values(
num_tokens * num_kv_heads, v_scale.item()
).view(num_tokens, kvsz)
qkv_orig = qkv.clone()
num_blocks = (num_tokens + block_size - 1) // block_size + 1
kv_cache = torch.zeros(
num_blocks,
2 * num_kv_heads,
block_size,
HEAD_DIM // 2 + HEAD_DIM // 16,
dtype=torch.uint8,
device=device,
)
index_cache = torch.zeros(
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
)
slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
slot_mapping[::5] = -1 # padded / unscheduled tokens are not written
index_slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
*weights[:2],
cos_sin,
positions,
num_heads,
num_kv_heads,
ROTARY_DIM,
eps,
*weights[2:],
num_idx_heads,
slot_mapping,
index_slot_mapping,
kv_cache,
index_cache,
block_size,
torch.empty(num_tokens, qsz, dtype=dtype, device=device),
torch.empty(num_tokens, iqsz, dtype=dtype, device=device),
kv_cache_dtype,
kv_k_scale=k_scale,
kv_v_scale=v_scale,
)
_, k_out, v_out, _, index_k = qkv.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)
torch.testing.assert_close(
v_out, qkv_orig.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)[2]
)
expected_kv_cache = nvfp4_pages(
[
(k_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / k_scale),
(v_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / v_scale),
],
slot_mapping,
num_blocks,
block_size,
)
assert torch.equal(kv_cache, expected_kv_cache), (
f"{int((kv_cache != expected_kv_cache).sum())} NVFP4 cache bytes differ"
)
expected_index_cache = torch.zeros_like(index_cache).view(-1, HEAD_DIM)
expected_index_cache[index_slot_mapping] = index_k
torch.testing.assert_close(
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
)