[Attention] Support NVFP4 KV cache on SM8x and SM12x with FlashInfer (#46963)

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
SeongJun Lee
2026-10-07 13:24:21 -04:00
committed by GitHub
co-authored by lesj0610 mgoin Claude
parent 3d6853b32c
commit 554340f3d3
2 changed files with 149 additions and 22 deletions
@@ -6,6 +6,7 @@ import pytest
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed from vllm.utils.torch_utils import set_random_seed
from vllm.v1.kv_cache_layout import KVCacheLayout
try: try:
import flashinfer import flashinfer
@@ -702,3 +703,35 @@ def test_flashinfer_decode_with_paged_fp8_kv(
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2), torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
f"{torch.max(torch.abs(output - ref_output))}", f"{torch.max(torch.abs(output - ref_output))}",
) )
@pytest.mark.skipif(
not any(current_platform.is_device_capability_family(f) for f in (80, 120)),
reason="NVFP4 fa2 path",
)
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
from vllm.v1.attention.backends import flashinfer as fi
torch.set_default_device("cuda")
order = KVCacheLayout[layout].stride_order
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
shape = (2, 6, 4, 16, 64 + 8)
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
layers = layers.permute(*(order.index(i) for i in range(5)))
one = torch.ones(())
layer = torch.nn.Module()
layer._k_scale = layer._v_scale = one
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
pages = torch.arange(6, dtype=torch.int32)[None]
seq_lens = torch.tensor([96], dtype=torch.int32)
deq = torch.empty_like(kv)[:, None]
flashinfer.nvfp4_kv_dequantize_paged(
data, scales, pages, seq_lens, one, one, *deq, kv_layout
)
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
assert not layers[0].any()
+116 -22
View File
@@ -16,6 +16,7 @@ from flashinfer import (
BatchPrefillWithRaggedKVCacheWrapper, BatchPrefillWithRaggedKVCacheWrapper,
MultiLevelCascadeAttentionWrapper, MultiLevelCascadeAttentionWrapper,
get_seq_lens, get_seq_lens,
nvfp4_quantize_append_paged_kv_cache_with_slot_mapping,
) )
from flashinfer.decode import fast_decode_plan, trtllm_batch_decode_with_kv_cache from flashinfer.decode import fast_decode_plan, trtllm_batch_decode_with_kv_cache
from flashinfer.prefill import trtllm_batch_context_with_kv_cache from flashinfer.prefill import trtllm_batch_context_with_kv_cache
@@ -106,6 +107,38 @@ logger = init_logger(__name__)
trtllm_workspace_buffer = None trtllm_workspace_buffer = None
def _nvfp4_kv_on_fa2() -> bool:
return current_platform.is_device_capability_family(
80
) or current_platform.is_device_capability_family(120)
_KVPair = tuple[torch.Tensor, torch.Tensor]
def _nvfp4_kv_views(
kv_cache: torch.Tensor, num_kv_heads: int
) -> tuple[_KVPair, _KVPair, str]:
"""Views of NVFP4 pages laid out as [K data | K scale | V data | V scale]."""
num_blocks, num_slots, block_size, full_dim = kv_cache.shape
hnd = kv_cache.stride()[1:] == (block_size * full_dim, full_dim, 1)
if not hnd and kv_cache.stride()[1:] != (full_dim, num_slots * full_dim, 1):
raise ValueError(f"NVFP4 KV pages must be dense, got {kv_cache.stride()}")
dims = (num_kv_heads, block_size) if hnd else (block_size, num_kv_heads)
side_size = num_kv_heads * block_size * full_dim
(k_data, k_scale), (v_data, v_scale) = (
nvfp4_split_data_scale(
kv_cache.as_strided(
(num_blocks, *dims, full_dim),
(kv_cache.stride(0), dims[1] * full_dim, full_dim, 1),
kv_cache.storage_offset() + side * side_size,
)
)
for side in range(2)
)
return (k_data, v_data), (k_scale, v_scale), "HND" if hnd else "NHD"
def _get_trtllm_workspace_buffer(): def _get_trtllm_workspace_buffer():
global trtllm_workspace_buffer global trtllm_workspace_buffer
if trtllm_workspace_buffer is None: if trtllm_workspace_buffer is None:
@@ -498,6 +531,9 @@ class FlashInferBackend(AttentionBackend):
@classmethod @classmethod
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool: def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
if kv_cache_dtype is not None and kv_cache_dtype.startswith("nvfp4"): if kv_cache_dtype is not None and kv_cache_dtype.startswith("nvfp4"):
if _nvfp4_kv_on_fa2():
# The slot-mapping writer has no scale search.
return kv_cache_dtype == "nvfp4"
return ( return (
current_platform.is_device_capability_family(100) current_platform.is_device_capability_family(100)
and supports_trtllm_attention(is_prefill=True) and supports_trtllm_attention(is_prefill=True)
@@ -505,6 +541,30 @@ class FlashInferBackend(AttentionBackend):
) )
return super().supports_kv_cache_dtype(kv_cache_dtype) return super().supports_kv_cache_dtype(kv_cache_dtype)
@classmethod
def supports_combination(
cls,
head_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType | None,
block_size: int | None,
use_mla: bool,
has_sink: bool,
use_sparse: bool,
use_mm_prefix: bool,
device_capability: DeviceCapability,
) -> str | None:
if (
kv_cache_dtype is not None
and kv_cache_dtype.startswith("nvfp4")
and device_capability.major in (8, 12)
):
if head_size == 64:
return "fa2 prefill misreads an NVFP4 KV cache at head_size 64"
if has_sink:
return "the sink prefill wrapper cannot read an NVFP4 KV cache"
return None
@classmethod @classmethod
def get_supported_head_sizes(cls) -> list[int]: def get_supported_head_sizes(cls) -> list[int]:
# https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157 # https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157
@@ -777,7 +837,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Cannot use self.kv_cache_spec.dtype here because kv_cache_spec # Cannot use self.kv_cache_spec.dtype here because kv_cache_spec
# storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3) # storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3)
self.is_kvcache_nvfp4 = self.cache_dtype.startswith("nvfp4") self.is_kvcache_nvfp4 = self.cache_dtype.startswith("nvfp4")
if self.is_kvcache_nvfp4: # fa2 plans NVFP4 by its uint8 storage dtype.
self.nvfp4_fa2 = self.is_kvcache_nvfp4 and _nvfp4_kv_on_fa2()
self.nvfp4_trtllm = self.is_kvcache_nvfp4 and not self.nvfp4_fa2
if self.nvfp4_fa2 and self.use_dcp:
raise NotImplementedError(
"The DCP prefill wrapper cannot read NVFP4 block scales."
)
if self.nvfp4_fa2 and self.head_dim == 64:
raise NotImplementedError("fa2 prefill misreads NVFP4 at head_size 64.")
if self.nvfp4_trtllm:
if ( if (
force_use_trtllm_attention() is False force_use_trtllm_attention() is False
or not supports_trtllm_attention(is_prefill=True) or not supports_trtllm_attention(is_prefill=True)
@@ -798,6 +867,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
else: else:
self.cache_dtype = "auto" self.cache_dtype = "auto"
self.is_kvcache_nvfp4 = False self.is_kvcache_nvfp4 = False
self.nvfp4_fa2 = self.nvfp4_trtllm = False
assert self.kv_cache_spec.dtype == self.model_config.dtype assert self.kv_cache_spec.dtype == self.model_config.dtype
self.kv_cache_dtype = self.kv_cache_spec.dtype self.kv_cache_dtype = self.kv_cache_spec.dtype
@@ -810,8 +880,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel # Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel
# must be selected statically for FULL cudagraph capture. # must be selected statically for FULL cudagraph capture.
can_use_xqa_or_trtllm_gen_decode = can_use_trtllm_attention( can_use_xqa_or_trtllm_gen_decode = (
self.num_qo_heads, self.num_kv_heads, is_prefill=False can_use_trtllm_attention(
self.num_qo_heads, self.num_kv_heads, is_prefill=False
)
and not self.nvfp4_fa2
) )
# Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by # Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by
# get_supported_kernel_block_sizes). # get_supported_kernel_block_sizes).
@@ -894,7 +967,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.window_left = self.global_hyperparameters.window_left self.window_left = self.global_hyperparameters.window_left
self.logits_soft_cap = self.global_hyperparameters.logits_soft_cap self.logits_soft_cap = self.global_hyperparameters.logits_soft_cap
self.has_sinks = self.global_hyperparameters.has_sinks self.has_sinks = self.global_hyperparameters.has_sinks
if self.has_sinks and not FlashInferBackend.supports_sink(): if self.has_sinks and (self.nvfp4_fa2 or not FlashInferBackend.supports_sink()):
raise NotImplementedError( raise NotImplementedError(
"FlashInfer backend currently does not support attention " "FlashInfer backend currently does not support attention "
"sinks, please use trtllm on blackwell or flash attention on " "sinks, please use trtllm on blackwell or flash attention on "
@@ -970,6 +1043,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
return FlashInferBackend.get_dtype_for_flashinfer(cache_dtype) return FlashInferBackend.get_dtype_for_flashinfer(cache_dtype)
return self.model_config.dtype return self.model_config.dtype
if cache_dtype.startswith("nvfp4"): if cache_dtype.startswith("nvfp4"):
if self.nvfp4_fa2:
return self.model_config.dtype
return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3") return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3")
return self.kv_cache_spec.dtype return self.kv_cache_spec.dtype
@@ -1002,7 +1077,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
num_qo_heads=num_qo_heads, num_qo_heads=num_qo_heads,
num_kv_heads=spec.num_kv_heads, num_kv_heads=spec.num_kv_heads,
is_prefill=False, is_prefill=False,
) or (is_xqa_arch and not _is_xqa_head_dim_supported(spec.head_size)): ) or (
is_xqa_arch
and (
spec.kv_quant_mode.is_nvfp4
or not _is_xqa_head_dim_supported(spec.head_size)
)
):
has_uniform_batch_support = False has_uniform_batch_support = False
break break
@@ -1230,8 +1311,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
) )
else: else:
# NVFP4 KV cache requires the trtllm-gen backend inside # NVFP4 KV cache requires the trtllm-gen backend inside
# the wrapper; fa2/fa3 do not support nvfp4. # the wrapper on SM100; SM8x and SM12x read it with fa2.
backend = "trtllm-gen" if self.is_kvcache_nvfp4 else "auto" backend = "trtllm-gen" if self.nvfp4_trtllm else "auto"
self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper( self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
self._get_workspace_buffer(), self._get_workspace_buffer(),
get_flashinfer_layout_string(self.kv_cache_layout), get_flashinfer_layout_string(self.kv_cache_layout),
@@ -1256,8 +1337,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
paged_kv_indices = None paged_kv_indices = None
paged_kv_last_page_len = None paged_kv_last_page_len = None
# NVFP4 KV cache requires the trtllm-gen backend inside # NVFP4 KV cache requires the trtllm-gen backend inside
# the wrapper; fa2/fa3 do not support nvfp4. # the wrapper on SM100; SM8x and SM12x read it with fa2.
backend = "trtllm-gen" if self.is_kvcache_nvfp4 else "auto" backend = "trtllm-gen" if self.nvfp4_trtllm else "auto"
decode_wrapper = BatchDecodeWithPagedKVCacheWrapper( decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
self._get_workspace_buffer(), self._get_workspace_buffer(),
get_flashinfer_layout_string(self.kv_cache_layout), get_flashinfer_layout_string(self.kv_cache_layout),
@@ -1710,7 +1791,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# use FP8 o_data_type so the wrapper matches the # use FP8 o_data_type so the wrapper matches the
# FP8 output buffer allocated in forward(). # FP8 output buffer allocated in forward().
o_dtype = ( o_dtype = (
FP8_DTYPE if self.is_kvcache_nvfp4 else self.model_config.dtype FP8_DTYPE if self.nvfp4_trtllm else self.model_config.dtype
) )
prefill_wrapper.plan( prefill_wrapper.plan(
qo_indptr=qo_indptr_prefill_cpu, qo_indptr=qo_indptr_prefill_cpu,
@@ -1797,9 +1878,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# NVFP4 trtllm kernel only supports FP8 output; # NVFP4 trtllm kernel only supports FP8 output;
# use FP8 o_data_type so the wrapper matches the # use FP8 o_data_type so the wrapper matches the
# FP8 output buffer allocated in forward(). # FP8 output buffer allocated in forward().
o_dtype = ( o_dtype = FP8_DTYPE if self.nvfp4_trtllm else self.model_config.dtype
FP8_DTYPE if self.is_kvcache_nvfp4 else self.model_config.dtype
)
paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_input_tokens + 1] paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_input_tokens + 1]
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[ paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[
:num_input_tokens :num_input_tokens
@@ -1891,6 +1970,8 @@ class FlashInferImpl(AttentionImpl):
) )
self.cache_dtype = kv_cache_dtype self.cache_dtype = kv_cache_dtype
self.is_kvcache_nvfp4 = kv_cache_dtype.startswith("nvfp4") self.is_kvcache_nvfp4 = kv_cache_dtype.startswith("nvfp4")
self.nvfp4_fa2 = self.is_kvcache_nvfp4 and _nvfp4_kv_on_fa2()
self.nvfp4_trtllm = self.is_kvcache_nvfp4 and not self.nvfp4_fa2
self.kv_cache_dtype = "nvfp4" if self.is_kvcache_nvfp4 else kv_cache_dtype self.kv_cache_dtype = "nvfp4" if self.is_kvcache_nvfp4 else kv_cache_dtype
self.fp4_data_dim = head_size // 2 if self.is_kvcache_nvfp4 else 0 self.fp4_data_dim = head_size // 2 if self.is_kvcache_nvfp4 else 0
self.logits_soft_cap = logits_soft_cap self.logits_soft_cap = logits_soft_cap
@@ -1937,7 +2018,7 @@ class FlashInferImpl(AttentionImpl):
self.o_sf_scale: float | None = None self.o_sf_scale: float | None = None
# Pre-allocated FP8 output buffer for NVFP4 without fused output quant. # Pre-allocated FP8 output buffer for NVFP4 without fused output quant.
if self.is_kvcache_nvfp4 and vllm_config is not None: if self.nvfp4_trtllm and vllm_config is not None:
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self._nvfp4_fp8_out = torch.empty( self._nvfp4_fp8_out = torch.empty(
(max_num_tokens, num_heads, head_size), (max_num_tokens, num_heads, head_size),
@@ -2195,10 +2276,9 @@ class FlashInferImpl(AttentionImpl):
canonicalize_singleton_dim_strides(k_cache.permute(*stride_order)), canonicalize_singleton_dim_strides(k_cache.permute(*stride_order)),
canonicalize_singleton_dim_strides(v_cache.permute(*stride_order)), canonicalize_singleton_dim_strides(v_cache.permute(*stride_order)),
) )
k_data, k_sf = nvfp4_split_data_scale(kv_cache_tuple[0]) nvfp4_kv_data, nvfp4_kv_block_scales, _ = _nvfp4_kv_views(
v_data, v_sf = nvfp4_split_data_scale(kv_cache_tuple[1]) kv_cache, self.num_kv_heads
nvfp4_kv_data = (k_data, v_data) )
nvfp4_kv_block_scales = (k_sf, v_sf)
else: else:
kv_cache_tuple = kv_cache_permute.split(hs, dim=-1) kv_cache_tuple = kv_cache_permute.split(hs, dim=-1)
@@ -2276,7 +2356,7 @@ class FlashInferImpl(AttentionImpl):
# Use a pre-allocated FP8 buffer and dequantize # Use a pre-allocated FP8 buffer and dequantize
# afterwards. # afterwards.
needs_fp8_out_prefill = ( needs_fp8_out_prefill = (
self.is_kvcache_nvfp4 and output.dtype != FP8_DTYPE self.nvfp4_trtllm and output.dtype != FP8_DTYPE
) )
if needs_fp8_out_prefill: if needs_fp8_out_prefill:
out_prefill = self._nvfp4_fp8_out[:num_prefill_tokens] out_prefill = self._nvfp4_fp8_out[:num_prefill_tokens]
@@ -2299,7 +2379,8 @@ class FlashInferImpl(AttentionImpl):
prefill_wrapper.run( prefill_wrapper.run(
prefill_query, prefill_query,
kv_cache_for_fi, kv_cache_for_fi,
q_scale=layer._q_scale_float, # The fa2 NVFP4 query stays in the model dtype.
q_scale=1.0 if self.nvfp4_fa2 else layer._q_scale_float,
k_scale=layer._k_scale_float, k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float, v_scale=layer._v_scale_float,
out=out_prefill, out=out_prefill,
@@ -2451,7 +2532,7 @@ class FlashInferImpl(AttentionImpl):
# NVFP4 kernel only supports FP8 output. # NVFP4 kernel only supports FP8 output.
# Use a pre-allocated FP8 buffer and dequantize afterwards. # Use a pre-allocated FP8 buffer and dequantize afterwards.
needs_fp8_out = self.is_kvcache_nvfp4 and output.dtype != FP8_DTYPE needs_fp8_out = self.nvfp4_trtllm and output.dtype != FP8_DTYPE
if needs_fp8_out: if needs_fp8_out:
out_decode = self._nvfp4_fp8_out[:num_decode_tokens] out_decode = self._nvfp4_fp8_out[:num_decode_tokens]
else: else:
@@ -2488,7 +2569,7 @@ class FlashInferImpl(AttentionImpl):
decode_wrapper.run( decode_wrapper.run(
decode_query, decode_query,
kv_cache_for_fi, kv_cache_for_fi,
q_scale=layer._q_scale_float, q_scale=1.0 if self.nvfp4_fa2 else layer._q_scale_float,
k_scale=layer._k_scale_float, k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float, v_scale=layer._v_scale_float,
out=out_decode, out=out_decode,
@@ -2668,6 +2749,19 @@ class FlashInferImpl(AttentionImpl):
# and value[:num_actual_tokens] because the reshape_and_cache_flash # and value[:num_actual_tokens] because the reshape_and_cache_flash
# op uses the slot_mapping's shape to determine the number of # op uses the slot_mapping's shape to determine the number of
# actual tokens. # actual tokens.
if self.nvfp4_fa2:
data, scales, kv_layout = _nvfp4_kv_views(kv_cache, self.num_kv_heads)
nvfp4_quantize_append_paged_kv_cache_with_slot_mapping(
key,
value,
slot_mapping,
data,
scales,
layer._k_scale,
layer._v_scale,
kv_layout=kv_layout,
)
return
if self.is_kvcache_nvfp4: if self.is_kvcache_nvfp4:
# (B, 2*H, N, full_dim) -> ((B, N, H, full_dim), # (B, 2*H, N, full_dim) -> ((B, N, H, full_dim),
# (B, N, H, full_dim)); # (B, N, H, full_dim));