diff --git a/tests/kernels/attention/test_flashinfer.py b/tests/kernels/attention/test_flashinfer.py index c6cba8d530..0d9aecd31e 100644 --- a/tests/kernels/attention/test_flashinfer.py +++ b/tests/kernels/attention/test_flashinfer.py @@ -6,6 +6,7 @@ import pytest from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed +from vllm.v1.kv_cache_layout import KVCacheLayout try: import flashinfer @@ -702,3 +703,35 @@ def test_flashinfer_decode_with_paged_fp8_kv( torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2), f"{torch.max(torch.abs(output - ref_output))}", ) + + +@pytest.mark.skipif( + not any(current_platform.is_device_capability_family(f) for f in (80, 120)), + reason="NVFP4 fa2 path", +) +@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"]) +def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout): + from vllm.v1.attention.backends import flashinfer as fi + + torch.set_default_device("cuda") + order = KVCacheLayout[layout].stride_order + # [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written. + shape = (2, 6, 4, 16, 64 + 8) + layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8) + layers = layers.permute(*(order.index(i) for i in range(5))) + one = torch.ones(()) + layer = torch.nn.Module() + layer._k_scale = layer._v_scale = one + kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16) + impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4") + impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96)) + + data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2) + pages = torch.arange(6, dtype=torch.int32)[None] + seq_lens = torch.tensor([96], dtype=torch.int32) + deq = torch.empty_like(kv)[:, None] + flashinfer.nvfp4_kv_dequantize_paged( + data, scales, pages, seq_lens, one, one, *deq, kv_layout + ) + assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2 + assert not layers[0].any() diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 91ff6da3f0..a21720ab26 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -16,6 +16,7 @@ from flashinfer import ( BatchPrefillWithRaggedKVCacheWrapper, MultiLevelCascadeAttentionWrapper, get_seq_lens, + nvfp4_quantize_append_paged_kv_cache_with_slot_mapping, ) from flashinfer.decode import fast_decode_plan, trtllm_batch_decode_with_kv_cache from flashinfer.prefill import trtllm_batch_context_with_kv_cache @@ -106,6 +107,38 @@ logger = init_logger(__name__) trtllm_workspace_buffer = None +def _nvfp4_kv_on_fa2() -> bool: + return current_platform.is_device_capability_family( + 80 + ) or current_platform.is_device_capability_family(120) + + +_KVPair = tuple[torch.Tensor, torch.Tensor] + + +def _nvfp4_kv_views( + kv_cache: torch.Tensor, num_kv_heads: int +) -> tuple[_KVPair, _KVPair, str]: + """Views of NVFP4 pages laid out as [K data | K scale | V data | V scale].""" + num_blocks, num_slots, block_size, full_dim = kv_cache.shape + hnd = kv_cache.stride()[1:] == (block_size * full_dim, full_dim, 1) + if not hnd and kv_cache.stride()[1:] != (full_dim, num_slots * full_dim, 1): + raise ValueError(f"NVFP4 KV pages must be dense, got {kv_cache.stride()}") + dims = (num_kv_heads, block_size) if hnd else (block_size, num_kv_heads) + side_size = num_kv_heads * block_size * full_dim + (k_data, k_scale), (v_data, v_scale) = ( + nvfp4_split_data_scale( + kv_cache.as_strided( + (num_blocks, *dims, full_dim), + (kv_cache.stride(0), dims[1] * full_dim, full_dim, 1), + kv_cache.storage_offset() + side * side_size, + ) + ) + for side in range(2) + ) + return (k_data, v_data), (k_scale, v_scale), "HND" if hnd else "NHD" + + def _get_trtllm_workspace_buffer(): global trtllm_workspace_buffer if trtllm_workspace_buffer is None: @@ -498,6 +531,9 @@ class FlashInferBackend(AttentionBackend): @classmethod def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool: if kv_cache_dtype is not None and kv_cache_dtype.startswith("nvfp4"): + if _nvfp4_kv_on_fa2(): + # The slot-mapping writer has no scale search. + return kv_cache_dtype == "nvfp4" return ( current_platform.is_device_capability_family(100) and supports_trtllm_attention(is_prefill=True) @@ -505,6 +541,30 @@ class FlashInferBackend(AttentionBackend): ) return super().supports_kv_cache_dtype(kv_cache_dtype) + @classmethod + def supports_combination( + cls, + head_size: int, + dtype: torch.dtype, + kv_cache_dtype: CacheDType | None, + block_size: int | None, + use_mla: bool, + has_sink: bool, + use_sparse: bool, + use_mm_prefix: bool, + device_capability: DeviceCapability, + ) -> str | None: + if ( + kv_cache_dtype is not None + and kv_cache_dtype.startswith("nvfp4") + and device_capability.major in (8, 12) + ): + if head_size == 64: + return "fa2 prefill misreads an NVFP4 KV cache at head_size 64" + if has_sink: + return "the sink prefill wrapper cannot read an NVFP4 KV cache" + return None + @classmethod def get_supported_head_sizes(cls) -> list[int]: # https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157 @@ -777,7 +837,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # Cannot use self.kv_cache_spec.dtype here because kv_cache_spec # storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3) self.is_kvcache_nvfp4 = self.cache_dtype.startswith("nvfp4") - if self.is_kvcache_nvfp4: + # fa2 plans NVFP4 by its uint8 storage dtype. + self.nvfp4_fa2 = self.is_kvcache_nvfp4 and _nvfp4_kv_on_fa2() + self.nvfp4_trtllm = self.is_kvcache_nvfp4 and not self.nvfp4_fa2 + if self.nvfp4_fa2 and self.use_dcp: + raise NotImplementedError( + "The DCP prefill wrapper cannot read NVFP4 block scales." + ) + if self.nvfp4_fa2 and self.head_dim == 64: + raise NotImplementedError("fa2 prefill misreads NVFP4 at head_size 64.") + if self.nvfp4_trtllm: if ( force_use_trtllm_attention() is False or not supports_trtllm_attention(is_prefill=True) @@ -798,6 +867,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): else: self.cache_dtype = "auto" self.is_kvcache_nvfp4 = False + self.nvfp4_fa2 = self.nvfp4_trtllm = False assert self.kv_cache_spec.dtype == self.model_config.dtype self.kv_cache_dtype = self.kv_cache_spec.dtype @@ -810,8 +880,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel # must be selected statically for FULL cudagraph capture. - can_use_xqa_or_trtllm_gen_decode = can_use_trtllm_attention( - self.num_qo_heads, self.num_kv_heads, is_prefill=False + can_use_xqa_or_trtllm_gen_decode = ( + can_use_trtllm_attention( + self.num_qo_heads, self.num_kv_heads, is_prefill=False + ) + and not self.nvfp4_fa2 ) # Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by # get_supported_kernel_block_sizes). @@ -894,7 +967,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self.window_left = self.global_hyperparameters.window_left self.logits_soft_cap = self.global_hyperparameters.logits_soft_cap self.has_sinks = self.global_hyperparameters.has_sinks - if self.has_sinks and not FlashInferBackend.supports_sink(): + if self.has_sinks and (self.nvfp4_fa2 or not FlashInferBackend.supports_sink()): raise NotImplementedError( "FlashInfer backend currently does not support attention " "sinks, please use trtllm on blackwell or flash attention on " @@ -970,6 +1043,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): return FlashInferBackend.get_dtype_for_flashinfer(cache_dtype) return self.model_config.dtype if cache_dtype.startswith("nvfp4"): + if self.nvfp4_fa2: + return self.model_config.dtype return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3") return self.kv_cache_spec.dtype @@ -1002,7 +1077,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): num_qo_heads=num_qo_heads, num_kv_heads=spec.num_kv_heads, is_prefill=False, - ) or (is_xqa_arch and not _is_xqa_head_dim_supported(spec.head_size)): + ) or ( + is_xqa_arch + and ( + spec.kv_quant_mode.is_nvfp4 + or not _is_xqa_head_dim_supported(spec.head_size) + ) + ): has_uniform_batch_support = False break @@ -1230,8 +1311,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): ) else: # NVFP4 KV cache requires the trtllm-gen backend inside - # the wrapper; fa2/fa3 do not support nvfp4. - backend = "trtllm-gen" if self.is_kvcache_nvfp4 else "auto" + # the wrapper on SM100; SM8x and SM12x read it with fa2. + backend = "trtllm-gen" if self.nvfp4_trtllm else "auto" self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper( self._get_workspace_buffer(), get_flashinfer_layout_string(self.kv_cache_layout), @@ -1256,8 +1337,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): paged_kv_indices = None paged_kv_last_page_len = None # NVFP4 KV cache requires the trtllm-gen backend inside - # the wrapper; fa2/fa3 do not support nvfp4. - backend = "trtllm-gen" if self.is_kvcache_nvfp4 else "auto" + # the wrapper on SM100; SM8x and SM12x read it with fa2. + backend = "trtllm-gen" if self.nvfp4_trtllm else "auto" decode_wrapper = BatchDecodeWithPagedKVCacheWrapper( self._get_workspace_buffer(), get_flashinfer_layout_string(self.kv_cache_layout), @@ -1710,7 +1791,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # use FP8 o_data_type so the wrapper matches the # FP8 output buffer allocated in forward(). o_dtype = ( - FP8_DTYPE if self.is_kvcache_nvfp4 else self.model_config.dtype + FP8_DTYPE if self.nvfp4_trtllm else self.model_config.dtype ) prefill_wrapper.plan( qo_indptr=qo_indptr_prefill_cpu, @@ -1797,9 +1878,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # NVFP4 trtllm kernel only supports FP8 output; # use FP8 o_data_type so the wrapper matches the # FP8 output buffer allocated in forward(). - o_dtype = ( - FP8_DTYPE if self.is_kvcache_nvfp4 else self.model_config.dtype - ) + o_dtype = FP8_DTYPE if self.nvfp4_trtllm else self.model_config.dtype paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_input_tokens + 1] paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[ :num_input_tokens @@ -1891,6 +1970,8 @@ class FlashInferImpl(AttentionImpl): ) self.cache_dtype = kv_cache_dtype self.is_kvcache_nvfp4 = kv_cache_dtype.startswith("nvfp4") + self.nvfp4_fa2 = self.is_kvcache_nvfp4 and _nvfp4_kv_on_fa2() + self.nvfp4_trtllm = self.is_kvcache_nvfp4 and not self.nvfp4_fa2 self.kv_cache_dtype = "nvfp4" if self.is_kvcache_nvfp4 else kv_cache_dtype self.fp4_data_dim = head_size // 2 if self.is_kvcache_nvfp4 else 0 self.logits_soft_cap = logits_soft_cap @@ -1937,7 +2018,7 @@ class FlashInferImpl(AttentionImpl): self.o_sf_scale: float | None = None # Pre-allocated FP8 output buffer for NVFP4 without fused output quant. - if self.is_kvcache_nvfp4 and vllm_config is not None: + if self.nvfp4_trtllm and vllm_config is not None: max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens self._nvfp4_fp8_out = torch.empty( (max_num_tokens, num_heads, head_size), @@ -2195,10 +2276,9 @@ class FlashInferImpl(AttentionImpl): canonicalize_singleton_dim_strides(k_cache.permute(*stride_order)), canonicalize_singleton_dim_strides(v_cache.permute(*stride_order)), ) - k_data, k_sf = nvfp4_split_data_scale(kv_cache_tuple[0]) - v_data, v_sf = nvfp4_split_data_scale(kv_cache_tuple[1]) - nvfp4_kv_data = (k_data, v_data) - nvfp4_kv_block_scales = (k_sf, v_sf) + nvfp4_kv_data, nvfp4_kv_block_scales, _ = _nvfp4_kv_views( + kv_cache, self.num_kv_heads + ) else: kv_cache_tuple = kv_cache_permute.split(hs, dim=-1) @@ -2276,7 +2356,7 @@ class FlashInferImpl(AttentionImpl): # Use a pre-allocated FP8 buffer and dequantize # afterwards. needs_fp8_out_prefill = ( - self.is_kvcache_nvfp4 and output.dtype != FP8_DTYPE + self.nvfp4_trtllm and output.dtype != FP8_DTYPE ) if needs_fp8_out_prefill: out_prefill = self._nvfp4_fp8_out[:num_prefill_tokens] @@ -2299,7 +2379,8 @@ class FlashInferImpl(AttentionImpl): prefill_wrapper.run( prefill_query, kv_cache_for_fi, - q_scale=layer._q_scale_float, + # The fa2 NVFP4 query stays in the model dtype. + q_scale=1.0 if self.nvfp4_fa2 else layer._q_scale_float, k_scale=layer._k_scale_float, v_scale=layer._v_scale_float, out=out_prefill, @@ -2451,7 +2532,7 @@ class FlashInferImpl(AttentionImpl): # NVFP4 kernel only supports FP8 output. # Use a pre-allocated FP8 buffer and dequantize afterwards. - needs_fp8_out = self.is_kvcache_nvfp4 and output.dtype != FP8_DTYPE + needs_fp8_out = self.nvfp4_trtllm and output.dtype != FP8_DTYPE if needs_fp8_out: out_decode = self._nvfp4_fp8_out[:num_decode_tokens] else: @@ -2488,7 +2569,7 @@ class FlashInferImpl(AttentionImpl): decode_wrapper.run( decode_query, kv_cache_for_fi, - q_scale=layer._q_scale_float, + q_scale=1.0 if self.nvfp4_fa2 else layer._q_scale_float, k_scale=layer._k_scale_float, v_scale=layer._v_scale_float, out=out_decode, @@ -2668,6 +2749,19 @@ class FlashInferImpl(AttentionImpl): # and value[:num_actual_tokens] because the reshape_and_cache_flash # op uses the slot_mapping's shape to determine the number of # actual tokens. + if self.nvfp4_fa2: + data, scales, kv_layout = _nvfp4_kv_views(kv_cache, self.num_kv_heads) + nvfp4_quantize_append_paged_kv_cache_with_slot_mapping( + key, + value, + slot_mapping, + data, + scales, + layer._k_scale, + layer._v_scale, + kv_layout=kv_layout, + ) + return if self.is_kvcache_nvfp4: # (B, 2*H, N, full_dim) -> ((B, N, H, full_dim), # (B, N, H, full_dim));