[Attention][MiniMax-M3] NVFP4 KV cache on the MSA sparse attention path (#59300)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ea84051819
commit
7566d83bd3
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for MiniMax M3 CUTLASS sparse decode."""
|
||||
"""Correctness tests for MiniMax M3 MSA sparse attention (CUTLASS decode and
|
||||
NVFP4 KV cache)."""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
@@ -8,6 +9,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.quantization.nvfp4_utils import dequant_nvfp4_kv_cache
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import AttentionConfig, CUDAGraphMode
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
@@ -18,6 +20,7 @@ from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseBackend,
|
||||
MiniMaxM3SparseMetadata,
|
||||
MiniMaxM3SparseMetadataBuilder,
|
||||
MiniMaxM3SparsePrefillMetadata,
|
||||
MiniMaxM3SparseTritonImpl,
|
||||
select_main_backend_and_impl_cls,
|
||||
)
|
||||
@@ -28,6 +31,7 @@ from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
|
||||
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
|
||||
MSACutlassDecodePlanCache,
|
||||
msa_cutlass_sparse_decode,
|
||||
nvfp4_kv_cache_views,
|
||||
prepare_decode_metadata,
|
||||
should_prepare_decode_metadata,
|
||||
)
|
||||
@@ -36,6 +40,7 @@ from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
|
||||
MiniMaxM3SparseMSADecodeMetadata,
|
||||
MiniMaxM3SparseMSAImpl,
|
||||
MiniMaxM3SparseMSAMetadataBuilder,
|
||||
MiniMaxM3SparseMSANvfp4Backend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -164,6 +169,53 @@ def test_msa_cutlass_decode_static_dispatch_requires_sm100(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode_backend", ["cutlass", "triton"])
|
||||
def test_msa_cutlass_decode_static_dispatch_nvfp4_has_no_fallback(
|
||||
decode_backend: str,
|
||||
) -> None:
|
||||
"""NVFP4 has no Triton decode, so it plans CUTLASS for any batch size."""
|
||||
assert should_prepare_decode_metadata(
|
||||
1,
|
||||
DEFAULT_QUERY_LEN,
|
||||
decode_backend=decode_backend, # type: ignore[arg-type]
|
||||
num_q_heads=16,
|
||||
num_kv_heads=1,
|
||||
kv_cache_dtype="nvfp4",
|
||||
page_size=BLOCK_SIZE,
|
||||
topk_blocks=TOPK,
|
||||
)
|
||||
|
||||
|
||||
def test_nvfp4_selects_msa_nvfp4_backend_and_requires_sm100(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
backend_cls, impl_cls = select_main_backend_and_impl_cls(
|
||||
topk_blocks=TOPK,
|
||||
kv_cache_dtype="nvfp4",
|
||||
num_kv_heads=1,
|
||||
)
|
||||
assert backend_cls is MiniMaxM3SparseMSANvfp4Backend
|
||||
assert impl_cls is MiniMaxM3SparseMSAImpl
|
||||
with pytest.raises(ValueError, match="nvfp4_4over6"):
|
||||
select_main_backend_and_impl_cls(
|
||||
topk_blocks=TOPK,
|
||||
kv_cache_dtype="nvfp4_4over6",
|
||||
num_kv_heads=1,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda _: False,
|
||||
)
|
||||
with pytest.raises(ValueError, match="NVFP4"):
|
||||
select_main_backend_and_impl_cls(
|
||||
topk_blocks=TOPK,
|
||||
kv_cache_dtype="nvfp4",
|
||||
num_kv_heads=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected"),
|
||||
[
|
||||
@@ -639,3 +691,391 @@ def test_msa_cutlass_decode_matches_triton_with_interleaved_cache(
|
||||
graph.replay()
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
|
||||
|
||||
|
||||
def _make_paged_layout(
|
||||
seq_lens_list: list[int],
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Block table over a shuffled physical page pool."""
|
||||
pages_per_request = [math.ceil(seq_len / BLOCK_SIZE) for seq_len in seq_lens_list]
|
||||
num_pages = sum(pages_per_request)
|
||||
block_table = torch.zeros(
|
||||
len(seq_lens_list), max(pages_per_request), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
physical_pages = torch.randperm(num_pages, dtype=torch.int32, device="cuda")
|
||||
offset = 0
|
||||
for request, request_pages in enumerate(pages_per_request):
|
||||
block_table[request, :request_pages] = physical_pages[
|
||||
offset : offset + request_pages
|
||||
]
|
||||
offset += request_pages
|
||||
return block_table, num_pages
|
||||
|
||||
|
||||
def _make_nvfp4_kv_cache(
|
||||
num_pages: int,
|
||||
num_kv_heads: int,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Write random K/V through the NVFP4 ``reshape_and_cache_flash`` writer,
|
||||
whose pages hold every head's data then every head's scales per side, and
|
||||
repack them into head slots.
|
||||
|
||||
Returns the packed HND cache ``[pages, 2 * Hkv, 128, 72]`` and its BF16
|
||||
dequantization ``[pages, Hkv, 128, 2 * 128]`` for the Triton reference.
|
||||
"""
|
||||
num_slots = num_pages * BLOCK_SIZE
|
||||
key = torch.randn(
|
||||
num_slots, num_kv_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
value = torch.randn_like(key)
|
||||
written = torch.zeros(
|
||||
num_pages,
|
||||
2 * num_kv_heads,
|
||||
BLOCK_SIZE,
|
||||
HEAD_DIM // 2 + HEAD_DIM // 16,
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
k_cache, v_cache = written.transpose(1, 2).split(num_kv_heads, dim=-2)
|
||||
ops.reshape_and_cache_flash(
|
||||
key,
|
||||
value,
|
||||
k_cache,
|
||||
v_cache,
|
||||
torch.arange(num_slots, dtype=torch.int64, device="cuda"),
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
|
||||
kv_cache = torch.empty_like(written)
|
||||
head_slot_views = iter(nvfp4_kv_cache_views(kv_cache))
|
||||
dequantized = []
|
||||
for side, global_scale, swizzled in (
|
||||
(written[:, :num_kv_heads], k_scale, False),
|
||||
(written[:, num_kv_heads:], v_scale, True),
|
||||
):
|
||||
data = side.as_strided(
|
||||
(num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2),
|
||||
(side.stride(0), BLOCK_SIZE * HEAD_DIM // 2, HEAD_DIM // 2, 1),
|
||||
)
|
||||
scale = side.as_strided(
|
||||
(num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 16),
|
||||
(side.stride(0), BLOCK_SIZE * HEAD_DIM // 16, HEAD_DIM // 16, 1),
|
||||
side.storage_offset() + num_kv_heads * BLOCK_SIZE * HEAD_DIM // 2,
|
||||
)
|
||||
next(head_slot_views).copy_(data)
|
||||
next(head_slot_views).copy_(scale)
|
||||
dequantized.append(
|
||||
dequant_nvfp4_kv_cache(
|
||||
data,
|
||||
scale,
|
||||
global_scale.item(),
|
||||
HEAD_DIM,
|
||||
BLOCK_SIZE,
|
||||
swizzled_scales=swizzled,
|
||||
)
|
||||
)
|
||||
return kv_cache, torch.cat(dequantized, dim=-1).to(torch.bfloat16)
|
||||
|
||||
|
||||
def test_nvfp4_kv_cache_views_address_head_slots() -> None:
|
||||
"""Slot 2h (K) and 2h + 1 (V) hold head h's data then scales, so each head
|
||||
is one contiguous run of the page: a TP rank's page is a byte slice."""
|
||||
num_kv_heads, slot_bytes = 4, BLOCK_SIZE * (HEAD_DIM // 2 + HEAD_DIM // 16)
|
||||
kv_cache = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(3, 2 * num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2 + HEAD_DIM // 16),
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache)
|
||||
runs = kv_cache.view(3, num_kv_heads, 2 * slot_bytes)
|
||||
for head in range(num_kv_heads):
|
||||
expected = torch.cat(
|
||||
[v[:, head].flatten(1) for v in (k_data, k_sf, v_data, v_sf)], dim=1
|
||||
)
|
||||
assert torch.equal(runs[:, head], expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_q_heads", "num_kv_heads", "num_request_pairs", "query_len", "capture"),
|
||||
[
|
||||
pytest.param(64, 4, 1, 1, True, id="tp1-small-batch-query-len-1"),
|
||||
pytest.param(64, 4, 8, 8, False, id="tp1-query-len-8"),
|
||||
pytest.param(16, 1, 16, 4, True, id="tp4-query-len-4"),
|
||||
],
|
||||
)
|
||||
def test_msa_cutlass_decode_nvfp4_matches_triton_on_dequantized_cache(
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
num_request_pairs: int,
|
||||
query_len: int,
|
||||
capture: bool,
|
||||
) -> None:
|
||||
"""CUTLASS decode reads vLLM's packed NVFP4 pages, including swizzled V
|
||||
scales and device global scales, as their exact dequantized values."""
|
||||
torch.manual_seed(0)
|
||||
seq_lens_list = [257, 513] * num_request_pairs
|
||||
seq_lens_cpu = torch.tensor(seq_lens_list, dtype=torch.int32)
|
||||
seq_lens = seq_lens_cpu.cuda()
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
|
||||
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
|
||||
kv_cache, kv_reference = _make_nvfp4_kv_cache(
|
||||
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
|
||||
)
|
||||
|
||||
num_query_tokens = len(seq_lens_list) * query_len
|
||||
query = torch.randn(
|
||||
num_query_tokens, num_q_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
query_fp8 = query.to(torch.float8_e4m3fn)
|
||||
topk = _make_topk(seq_lens_list, num_kv_heads, query_len)
|
||||
expected = torch.empty_like(query)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
query_fp8.to(torch.bfloat16),
|
||||
kv_reference,
|
||||
topk.transpose(0, 1),
|
||||
block_table,
|
||||
seq_lens,
|
||||
num_kv_heads,
|
||||
SM_SCALE,
|
||||
expected,
|
||||
query_len,
|
||||
)
|
||||
|
||||
assert should_prepare_decode_metadata(
|
||||
len(seq_lens_list),
|
||||
query_len,
|
||||
decode_backend="triton",
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
kv_cache_dtype="nvfp4",
|
||||
page_size=BLOCK_SIZE,
|
||||
topk_blocks=TOPK,
|
||||
)
|
||||
metadata = prepare_decode_metadata(
|
||||
block_table,
|
||||
seq_lens,
|
||||
seq_lens_cpu,
|
||||
query_len,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
page_size=BLOCK_SIZE,
|
||||
topk_blocks=TOPK,
|
||||
)
|
||||
actual = torch.zeros_like(query)
|
||||
|
||||
def run() -> None:
|
||||
msa_cutlass_sparse_decode(
|
||||
query_fp8,
|
||||
kv_cache,
|
||||
topk,
|
||||
actual,
|
||||
metadata,
|
||||
scale=SM_SCALE,
|
||||
q_scale_float=1.0,
|
||||
k_scale_float=1.0,
|
||||
v_scale_float=1.0,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
|
||||
if capture:
|
||||
run()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
actual.zero_()
|
||||
graph.replay()
|
||||
else:
|
||||
run()
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_q_heads", "num_kv_heads"),
|
||||
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
|
||||
)
|
||||
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
) -> None:
|
||||
"""MSA prefill reads NVFP4 pages as their exact dequantized values,
|
||||
including chunked prefill over cached context, and takes q from its fp8
|
||||
copy (the only one the fused insert writes on this path)."""
|
||||
torch.manual_seed(0)
|
||||
q_lens = [300, 128, 77]
|
||||
seq_lens_list = [300, 640, 1100]
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
|
||||
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
|
||||
kv_cache, kv_reference = _make_nvfp4_kv_cache(
|
||||
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
|
||||
)
|
||||
|
||||
total_q = sum(q_lens)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
|
||||
context_lens = seq_lens - torch.tensor(q_lens, dtype=torch.int32, device="cuda")
|
||||
cu_seqlens_q = torch.zeros(len(q_lens) + 1, dtype=torch.int32, device="cuda")
|
||||
cu_seqlens_q[1:] = torch.tensor(q_lens, device="cuda").cumsum(0)
|
||||
cu_seqlens_k = torch.zeros_like(cu_seqlens_q)
|
||||
cu_seqlens_k[1:] = seq_lens.cumsum(0)
|
||||
prefill = MiniMaxM3SparsePrefillMetadata(
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
cu_seqlens_k=cu_seqlens_k,
|
||||
seq_lens=seq_lens,
|
||||
context_lens=context_lens,
|
||||
block_table=block_table,
|
||||
max_query_len=max(q_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
total_kv_blocks=sum(math.ceil(n / BLOCK_SIZE) for n in seq_lens_list),
|
||||
)
|
||||
metadata = MiniMaxM3SparseMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=max(seq_lens_list),
|
||||
slot_mapping=torch.empty(total_q, dtype=torch.int64, device="cuda"),
|
||||
num_actual_tokens=total_q,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_prefills=len(q_lens),
|
||||
num_prefill_tokens=total_q,
|
||||
prefill=prefill,
|
||||
)
|
||||
|
||||
# Every causally visible page (<= TOPK per query), ending with the local one.
|
||||
topk = torch.full(
|
||||
(total_q, num_kv_heads, TOPK), -1, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
token = 0
|
||||
for q_len, seq_len in zip(q_lens, seq_lens_list):
|
||||
for local_q in range(q_len):
|
||||
visible = (seq_len - q_len + local_q) // BLOCK_SIZE + 1
|
||||
assert visible <= TOPK
|
||||
topk[token, :, :visible] = torch.arange(visible, device="cuda")
|
||||
token += 1
|
||||
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
q_scale = 0.5
|
||||
layer = SimpleNamespace(
|
||||
layer_name=layer_name,
|
||||
topk_indices_buffer=topk,
|
||||
_q_scale_float=q_scale,
|
||||
_k_scale=k_scale,
|
||||
_v_scale=v_scale,
|
||||
)
|
||||
query_fp8 = (
|
||||
torch.randn(
|
||||
total_q, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
/ q_scale
|
||||
).to(torch.float8_e4m3fn)
|
||||
query = query_fp8.to(torch.bfloat16) * q_scale
|
||||
common = dict(
|
||||
num_heads=num_q_heads,
|
||||
head_size=HEAD_DIM,
|
||||
scale=SM_SCALE,
|
||||
num_kv_heads=num_kv_heads,
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
)
|
||||
forward_context = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={layer_name: metadata},
|
||||
slot_mapping={},
|
||||
)
|
||||
with override_forward_context(forward_context):
|
||||
expected = MiniMaxM3SparseTritonImpl(**common).forward(
|
||||
layer, query, kv_reference, torch.empty_like(query)
|
||||
)
|
||||
actual = MiniMaxM3SparseMSAImpl(**common, kv_cache_dtype="nvfp4").forward(
|
||||
layer,
|
||||
torch.full_like(query, float("nan")),
|
||||
kv_cache,
|
||||
torch.zeros_like(query),
|
||||
query_fp8=query_fp8,
|
||||
)
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
|
||||
|
||||
|
||||
def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
|
||||
"""With CUTLASS decode on but not planned for the batch, the Triton decode
|
||||
fallback attends with the dequantized fp8 q, the only q written."""
|
||||
torch.manual_seed(0)
|
||||
num_q_heads, num_kv_heads, q_scale = 16, 1, 0.5
|
||||
seq_lens_list = [257, 513, 129]
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
kv_cache = (
|
||||
torch.randn(
|
||||
num_pages,
|
||||
num_kv_heads,
|
||||
BLOCK_SIZE,
|
||||
2 * HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
* 0.25
|
||||
).to(torch.float8_e4m3fn)
|
||||
num_tokens = len(seq_lens_list)
|
||||
metadata = MiniMaxM3SparseMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=max(seq_lens_list),
|
||||
slot_mapping=torch.empty(num_tokens, dtype=torch.int64, device="cuda"),
|
||||
num_actual_tokens=num_tokens,
|
||||
num_decodes=num_tokens,
|
||||
num_decode_tokens=num_tokens,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
decode=MiniMaxM3SparseMSADecodeMetadata(
|
||||
seq_lens=seq_lens,
|
||||
block_table=block_table,
|
||||
decode_query_len=1,
|
||||
msa_cutlass=None,
|
||||
),
|
||||
)
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
layer = SimpleNamespace(
|
||||
layer_name=layer_name,
|
||||
topk_indices_buffer=_make_topk(seq_lens_list, num_kv_heads, 1),
|
||||
_q_scale_float=q_scale,
|
||||
)
|
||||
query_fp8 = torch.randn(
|
||||
num_tokens, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
).to(torch.float8_e4m3fn)
|
||||
query = query_fp8.to(torch.bfloat16) * q_scale
|
||||
common = dict(
|
||||
num_heads=num_q_heads,
|
||||
head_size=HEAD_DIM,
|
||||
scale=SM_SCALE,
|
||||
num_kv_heads=num_kv_heads,
|
||||
kv_cache_dtype="fp8",
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
)
|
||||
impl = MiniMaxM3SparseMSAImpl(**common, msa_decode_backend="cutlass")
|
||||
assert impl.use_cutlass_decode
|
||||
forward_context = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={layer_name: metadata},
|
||||
slot_mapping={},
|
||||
)
|
||||
with override_forward_context(forward_context):
|
||||
expected = MiniMaxM3SparseTritonImpl(**common).forward(
|
||||
layer, query, kv_cache, torch.empty_like(query)
|
||||
)
|
||||
actual = impl.forward(
|
||||
layer,
|
||||
torch.full_like(query, float("nan")),
|
||||
kv_cache,
|
||||
torch.zeros_like(query),
|
||||
query_fp8=query_fp8,
|
||||
)
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
|
||||
|
||||
@@ -320,6 +320,38 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
|
||||
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
|
||||
)
|
||||
|
||||
# Without q_out, q is emitted only in fp8: the same bytes, while qkv keeps
|
||||
# its raw q and every other output matches the run above.
|
||||
outputs = [qkv_orig.clone(), torch.empty_like(q_fp8), torch.empty_like(index_q)]
|
||||
caches = [torch.zeros_like(kv_cache), torch.zeros_like(index_cache)]
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
outputs[0],
|
||||
q_w,
|
||||
k_w,
|
||||
cos_sin,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
ROTARY_DIM,
|
||||
eps,
|
||||
iq_w,
|
||||
ik_w,
|
||||
num_idx_heads,
|
||||
slot_mapping,
|
||||
index_slot_mapping,
|
||||
*caches,
|
||||
block_size,
|
||||
None,
|
||||
outputs[2],
|
||||
kv_cache_dtype,
|
||||
q_fp8_out=outputs[1],
|
||||
q_fp8_scale=0.5,
|
||||
)
|
||||
for actual, expected in zip(
|
||||
outputs + caches, [qkv, q_fp8, index_q, kv_cache, index_cache]
|
||||
):
|
||||
assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 64, 513])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
@@ -441,8 +473,8 @@ def test_sparse_skip_index_branch(num_tokens, block_size, kv_cache_dtype):
|
||||
# ── Test 4: fp8 (e4m3) index outputs ─────────────────────────────────────────
|
||||
# The fp8 score path stores index_q and the index-K cache as e4m3 while q/k/v +
|
||||
# q_out stay bf16. Asserts: (1) q/k/v/q_out are bit-identical to the bf16 run
|
||||
# (the index dtype must not perturb the main branch), and (2) the e4m3 index
|
||||
# outputs dequantize close to the bf16 reference.
|
||||
# (the index dtype must not perturb the main branch), and (2) index_q is the
|
||||
# e4m3 cast of the bf16 run's index_q, and the index-K cache is within fp8 ulp.
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
@@ -528,6 +560,196 @@ def test_sparse_full_fp8_index(num_tokens, block_size):
|
||||
torch.testing.assert_close(qkv_fp, qkv_bf, rtol=0, atol=0)
|
||||
torch.testing.assert_close(kvc_fp, kvc_bf, rtol=0, atol=0)
|
||||
|
||||
# (2) Dequantized e4m3 index outputs match the bf16 reference within fp8 ulp.
|
||||
torch.testing.assert_close(iq_fp.float(), iq_bf.float(), rtol=0.13, atol=0.05)
|
||||
# (2) index_q is e4m3(bf16(q)); the index-K cache is within fp8 ulp.
|
||||
assert torch.equal(
|
||||
iq_fp.view(torch.uint8), iq_bf.to(torch.float8_e4m3fn).view(torch.uint8)
|
||||
)
|
||||
torch.testing.assert_close(idxc_fp.float(), idxc_bf.float(), rtol=0.13, atol=0.05)
|
||||
|
||||
|
||||
# ── Test 5: sparse mode with an NVFP4 main KV cache ──────────────────────────
|
||||
# The NVFP4 inserts quantize x * sf_scale, where sf_scale = 1 / the global
|
||||
# scale: per 16-value block,
|
||||
# scale = min(448, max(E4M3(max(amax, 1e-12) / 6), 1/512)) and
|
||||
# code = sign | RNE E2M1 of min(|x / scale|, 6), with zero stored as +0.
|
||||
|
||||
E2M1_MIDPOINTS = (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)
|
||||
|
||||
|
||||
def _div(a, b):
|
||||
# Division by a Python scalar would become a reciprocal multiply.
|
||||
return a / torch.as_tensor(b, dtype=a.dtype, device=a.device)
|
||||
|
||||
|
||||
def nvfp4_ref(x, sf_scale):
|
||||
"""Returns the E2M1 codes [..., 128] and E4M3 scale bytes [..., 8] of x."""
|
||||
blocks = x.float().unflatten(-1, (-1, 16))
|
||||
amax = blocks.abs().amax(-1, keepdim=True).clamp_min(1e-12)
|
||||
scale = (sf_scale * _div(amax, 6.0)).clamp(max=448.0)
|
||||
scale = scale.to(torch.float8_e4m3fn).float().clamp(min=2.0**-9)
|
||||
q = _div(blocks * sf_scale, scale)
|
||||
# Alternating > / >= rounds exact midpoints to the even code.
|
||||
mag = sum(
|
||||
(q.abs() >= m) if i % 2 else (q.abs() > m) for i, m in enumerate(E2M1_MIDPOINTS)
|
||||
).to(torch.uint8)
|
||||
codes = torch.where(mag > 0, mag | (torch.signbit(q).to(torch.uint8) << 3), 0)
|
||||
scale_bytes = scale.squeeze(-1).to(torch.float8_e4m3fn).view(torch.uint8)
|
||||
return codes.flatten(-2), scale_bytes
|
||||
|
||||
|
||||
def nvfp4_pages(sides, slot_mapping, num_blocks, block_size):
|
||||
"""Expected NVFP4 pages [nb, len(sides) * nkv, bs, 72] for the K (then V)
|
||||
side given as (x [N, nkv, 128], sf_scale). Slot h * len(sides) + side is
|
||||
head h's packed E2M1 data [bs, 64] (low nibble = even element), then its
|
||||
E4M3 scales [bs, 8]: linear for K, 4x4 token-quad swizzled for V."""
|
||||
nkv, ns, device = sides[0][0].shape[1], HEAD_DIM // 16, slot_mapping.device
|
||||
row_bytes, data_bytes = HEAD_DIM // 2 + ns, block_size * HEAD_DIM // 2
|
||||
pages = torch.zeros(
|
||||
num_blocks,
|
||||
nkv,
|
||||
len(sides),
|
||||
block_size * row_bytes,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
valid = slot_mapping >= 0
|
||||
blk, tok = slot_mapping[valid] // block_size, slot_mapping[valid] % block_size
|
||||
g, t = torch.arange(ns, device=device), tok[:, None]
|
||||
scale_offsets = (t * ns + g, ((t // 4) * 4 + g // 2) * ns + (g % 2) * 4 + t % 4)
|
||||
for side, (x, sf_scale) in enumerate(sides):
|
||||
codes, scales = nvfp4_ref(x[valid], sf_scale)
|
||||
data = pages[:, :, side, :data_bytes].view(num_blocks, nkv, block_size, -1)
|
||||
data[blk, :, tok] = codes[..., 0::2] | (codes[..., 1::2] << 4)
|
||||
scale_region = pages[:, :, side, data_bytes:]
|
||||
for h in range(nkv):
|
||||
scale_region[blk[:, None], h, scale_offsets[side]] = scales[:, h]
|
||||
return pages.view(num_blocks, -1, block_size, row_bytes)
|
||||
|
||||
|
||||
def nvfp4_edge_values(num_rows, global_scale):
|
||||
"""[num_rows, 128] bf16 whose 16-value blocks, divided by global_scale, hit
|
||||
the quantizer's corner cases: exact E2M1 midpoints under non-power-of-2 scales
|
||||
(blocks 0-1), amax under the 1/512 scale floor (2), all -0 (3), and
|
||||
saturation with |x| > 4096 up to bf16 max (4) or with inf (5)."""
|
||||
g = torch.Generator().manual_seed(0)
|
||||
x = torch.randn(num_rows, 8, 16, generator=g)
|
||||
sign = torch.where(torch.rand(x.shape, generator=g) < 0.5, -1.0, 1.0)
|
||||
steps = torch.tensor(E2M1_MIDPOINTS + (0.5, 1.0, 3.0))
|
||||
mant = 1 + torch.randint(1, 8, (num_rows, 2, 1), generator=g) / 8
|
||||
scale = mant * 2.0 ** torch.randint(-6, 3, (num_rows, 2, 1), generator=g)
|
||||
x[:, :2] = steps[torch.randint(len(steps), (num_rows, 2, 16), generator=g)] * scale
|
||||
x[:, :2, :1] = 6 * scale # amax = 6 * scale, so the block scale is `scale`
|
||||
x[:, 2] = torch.rand(num_rows, 16, generator=g) * 2.0**-8
|
||||
x[:, 4] *= 5000
|
||||
x = x * sign * global_scale
|
||||
x[:, 3] = -0.0
|
||||
x[:, 4, 0] = sign[:, 4, 0] * torch.finfo(torch.bfloat16).max
|
||||
x[:, 5, 0] = sign[:, 5, 0] * float("inf")
|
||||
return x.flatten(1).to(torch.bfloat16)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
|
||||
reason="NVFP4 KV caches need SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 4])
|
||||
@pytest.mark.parametrize("block_size", [16, 128])
|
||||
@pytest.mark.parametrize("kv_scales", [(0.5, 2.0), (1.0, 1.0)])
|
||||
def test_sparse_full_nvfp4(num_tokens, num_kv_heads, block_size, kv_scales):
|
||||
"""The fused NVFP4 insert writes the reference quantizer's bytes (packed
|
||||
E2M1, linear K / swizzled V E4M3 block scales) for the kernel's own
|
||||
post-RoPE k and for a raw v that hits the quantizer's corner cases, and
|
||||
skips padded slots. Unit global scales take the kernel's separate fast
|
||||
path."""
|
||||
kv_cache_dtype = "nvfp4"
|
||||
torch.manual_seed(5)
|
||||
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
|
||||
base, max_pos = 5_000_000.0, 4096
|
||||
num_heads, num_idx_heads = 4 * num_kv_heads, 1
|
||||
|
||||
weights = [
|
||||
torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1 for _ in range(4)
|
||||
]
|
||||
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
|
||||
iqsz = num_idx_heads * HEAD_DIM
|
||||
qkv = torch.randn(
|
||||
num_tokens, qsz + 2 * kvsz + iqsz + HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
k_scale, v_scale = (
|
||||
torch.tensor(s, dtype=torch.float32, device=device) for s in kv_scales
|
||||
)
|
||||
qkv[:, qsz + kvsz : qsz + 2 * kvsz] = nvfp4_edge_values(
|
||||
num_tokens * num_kv_heads, v_scale.item()
|
||||
).view(num_tokens, kvsz)
|
||||
qkv_orig = qkv.clone()
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
2 * num_kv_heads,
|
||||
block_size,
|
||||
HEAD_DIM // 2 + HEAD_DIM // 16,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
index_cache = torch.zeros(
|
||||
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
slot_mapping = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device=device
|
||||
)[:num_tokens]
|
||||
slot_mapping[::5] = -1 # padded / unscheduled tokens are not written
|
||||
index_slot_mapping = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device=device
|
||||
)[:num_tokens]
|
||||
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
*weights[:2],
|
||||
cos_sin,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
ROTARY_DIM,
|
||||
eps,
|
||||
*weights[2:],
|
||||
num_idx_heads,
|
||||
slot_mapping,
|
||||
index_slot_mapping,
|
||||
kv_cache,
|
||||
index_cache,
|
||||
block_size,
|
||||
torch.empty(num_tokens, qsz, dtype=dtype, device=device),
|
||||
torch.empty(num_tokens, iqsz, dtype=dtype, device=device),
|
||||
kv_cache_dtype,
|
||||
kv_k_scale=k_scale,
|
||||
kv_v_scale=v_scale,
|
||||
)
|
||||
|
||||
_, k_out, v_out, _, index_k = qkv.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)
|
||||
torch.testing.assert_close(
|
||||
v_out, qkv_orig.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)[2]
|
||||
)
|
||||
expected_kv_cache = nvfp4_pages(
|
||||
[
|
||||
(k_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / k_scale),
|
||||
(v_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / v_scale),
|
||||
],
|
||||
slot_mapping,
|
||||
num_blocks,
|
||||
block_size,
|
||||
)
|
||||
assert torch.equal(kv_cache, expected_kv_cache), (
|
||||
f"{int((kv_cache != expected_kv_cache).sum())} NVFP4 cache bytes differ"
|
||||
)
|
||||
|
||||
expected_index_cache = torch.zeros_like(index_cache).view(-1, HEAD_DIM)
|
||||
expected_index_cache[index_slot_mapping] = index_k
|
||||
torch.testing.assert_close(
|
||||
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user