[Attention] Support NVFP4 KV cache on SM8x and SM12x with FlashInfer (#46963)

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
SeongJun Lee
2026-10-07 13:24:21 -04:00
committed by GitHub
co-authored by lesj0610 mgoin Claude
parent 3d6853b32c
commit 554340f3d3
2 changed files with 149 additions and 22 deletions
@@ -6,6 +6,7 @@ import pytest
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.kv_cache_layout import KVCacheLayout
try:
import flashinfer
@@ -702,3 +703,35 @@ def test_flashinfer_decode_with_paged_fp8_kv(
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}",
)
@pytest.mark.skipif(
not any(current_platform.is_device_capability_family(f) for f in (80, 120)),
reason="NVFP4 fa2 path",
)
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
from vllm.v1.attention.backends import flashinfer as fi
torch.set_default_device("cuda")
order = KVCacheLayout[layout].stride_order
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
shape = (2, 6, 4, 16, 64 + 8)
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
layers = layers.permute(*(order.index(i) for i in range(5)))
one = torch.ones(())
layer = torch.nn.Module()
layer._k_scale = layer._v_scale = one
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
pages = torch.arange(6, dtype=torch.int32)[None]
seq_lens = torch.tensor([96], dtype=torch.int32)
deq = torch.empty_like(kv)[:, None]
flashinfer.nvfp4_kv_dequantize_paged(
data, scales, pages, seq_lens, one, one, *deq, kv_layout
)
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
assert not layers[0].any()