[Attention] Support NVFP4 KV cache on SM8x and SM12x with FlashInfer (#46963)
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
lesj0610
mgoin
Claude
parent
3d6853b32c
commit
554340f3d3
@@ -6,6 +6,7 @@ import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
try:
|
||||
import flashinfer
|
||||
@@ -702,3 +703,35 @@ def test_flashinfer_decode_with_paged_fp8_kv(
|
||||
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not any(current_platform.is_device_capability_family(f) for f in (80, 120)),
|
||||
reason="NVFP4 fa2 path",
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
|
||||
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
order = KVCacheLayout[layout].stride_order
|
||||
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
|
||||
shape = (2, 6, 4, 16, 64 + 8)
|
||||
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
|
||||
layers = layers.permute(*(order.index(i) for i in range(5)))
|
||||
one = torch.ones(())
|
||||
layer = torch.nn.Module()
|
||||
layer._k_scale = layer._v_scale = one
|
||||
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
|
||||
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
|
||||
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
|
||||
|
||||
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
|
||||
pages = torch.arange(6, dtype=torch.int32)[None]
|
||||
seq_lens = torch.tensor([96], dtype=torch.int32)
|
||||
deq = torch.empty_like(kv)[:, None]
|
||||
flashinfer.nvfp4_kv_dequantize_paged(
|
||||
data, scales, pages, seq_lens, one, one, *deq, kv_layout
|
||||
)
|
||||
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
|
||||
assert not layers[0].any()
|
||||
|
||||
Reference in New Issue
Block a user