From 9bd074a99631a87095465590362f63dda5997ba1 Mon Sep 17 00:00:00 2001 From: SeongJun Lee Date: Sat, 10 Oct 2026 13:24:00 +0900 Subject: [PATCH] [Attention] Decode an NVFP4 KV cache with XQA on SM12x (#60452) Signed-off-by: lesj0610 Co-authored-by: Misha Goin --- .buildkite/test_areas/attention.yaml | 14 +++ tests/v1/attention/test_attention_backends.py | 96 ++++++++++++++++++- vllm/v1/attention/backends/flashinfer.py | 21 ++-- 3 files changed, 121 insertions(+), 10 deletions(-) diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index b9f430a20e..83ba90e2cb 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -62,3 +62,17 @@ steps: commands: - pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + +- label: ":nvidia: (DGX) Spark FlashInfer NVFP4 XQA" + key: flashinfer-nvfp4-xqa-dgx-spark + timeout_in_minutes: 30 + device: dgx-spark + optional: true + num_devices: 1 + depends_on: + - arm64-image-build + source_file_dependencies: + - vllm/v1/attention/backends/flashinfer.py + - tests/v1/attention/test_attention_backends.py + commands: + - pytest -v -s v1/attention/test_attention_backends.py -k test_flashinfer_xqa_nvfp4_decode_correctness diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index 77366c02ae..add611d92b 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -31,6 +31,7 @@ from vllm.utils.torch_utils import ( STR_DTYPE_TO_TORCH_DTYPE, is_quantized_kv_cache, is_torch_equal_or_newer, + nvfp4_kv_cache_full_dim, set_random_seed, ) from vllm.v1.attention.backend import ( @@ -41,7 +42,7 @@ from vllm.v1.attention.backend import ( max_decode_query_len, ) from vllm.v1.attention.backends.registry import AttentionBackendEnum -from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout +from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode BACKENDS_TO_TEST = [ AttentionBackendEnum.FLASH_ATTN, @@ -1698,6 +1699,99 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config): ) +@pytest.mark.skipif( + AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST + or not current_platform.is_device_capability_family(120), + reason="NVFP4 XQA decode requires SM12x.", +) +@pytest.mark.parametrize("use_cuda_graph", [False, True]) +@pytest.mark.parametrize("q_len", [1, 4]) +def test_flashinfer_xqa_nvfp4_decode_correctness( + default_vllm_config, q_len, use_cuda_graph +): + """Single-token and spec-decode queries over an NVFP4 cache decode with + XQA, eager and replayed, and match the fa2 prefill that wrote the cache.""" + import unittest.mock + + from vllm.v1.attention.backends import flashinfer as fi + + seq_lens, bs, d = [40, 300, 1027], 16, 256 + config = create_vllm_config(max_model_len=max(seq_lens), block_size=bs) + # GQA 6 at head_dim 256, as Qwen3.8-27B at TP=2. + config.model_config.model_arch_config = replace( + config.model_config.model_arch_config, + total_num_attention_heads=24, + total_num_kv_heads=4, + head_size=d, + ) + config.cache_config.cache_dtype = "nvfp4" + if q_len > 1: + config.speculative_config = SpeculativeConfig( + method="ngram", num_speculative_tokens=q_len - 1 + ) + spec = FullAttentionSpec( + block_size=bs, + num_kv_heads=4, + head_size=d, + dtype=torch.uint8, + kv_quant_mode=KVQuantMode.NVFP4, + ) + cg_support = fi.FlashInferMetadataBuilder.get_cudagraph_support(config, spec) + assert cg_support == AttentionCGSupport.UNIFORM_BATCH + + device = torch.device(f"{DEVICE_TYPE}:0") + # Request i owns consecutive pages from starts[i]; page 0 stays null. + pages = [0] + [cdiv(s, bs) for s in seq_lens[:-1]] + starts = 1 + torch.tensor(pages).cumsum(0) + max_pages = cdiv(max(seq_lens), bs) + block_table = (starts[:, None] + torch.arange(max_pages)).int().to(device) + full_dim = nvfp4_kv_cache_full_dim(d) + kv_cache = torch.zeros( + int(starts[-1]) + max_pages, bs, 8, full_dim, dtype=torch.uint8, device=device + ).transpose(1, 2) + q, k, v = ( + torch.randn(sum(seq_lens), h, d).bfloat16().to(device) for h in (24, 4, 4) + ) + + def run(query_lens, use_cuda_graph=False): + req = torch.repeat_interleave(torch.arange(3), torch.tensor(query_lens)) + pos = torch.cat([torch.arange(s - n, s) for s, n in zip(seq_lens, query_lens)]) + rows = pos + torch.tensor([0, *seq_lens]).cumsum(0)[req] + metadata = replace( + create_common_attn_metadata(BatchSpec(seq_lens, query_lens), bs, device), + block_table_tensor=block_table, + slot_mapping=(starts[req] * bs + pos).to(device), + ) + with set_current_vllm_config(config): + return rows, run_attention_backend( + AttentionBackendEnum.FLASHINFER, + spec, + ["placeholder"], + config, + device, + metadata, + q[rows], + k[rows], + v[rows], + kv_cache, + kv_cache_dtype="nvfp4", + use_cuda_graph=use_cuda_graph, + layer_k_scale=0.02, + layer_v_scale=0.03, + ) + + # The full prefill writes every token through the NVFP4 writer; its fa2 + # output for the last q_len tokens of each request is the reference. + _, reference = run(seq_lens) + xqa_fn = fi.flashinfer_xqa_batch_decode_with_kv_cache + with unittest.mock.patch.object( + fi, "flashinfer_xqa_batch_decode_with_kv_cache", wraps=xqa_fn + ) as xqa: + rows, output = run([q_len] * len(seq_lens), use_cuda_graph) + assert xqa.called + torch.testing.assert_close(output, reference[rows], atol=2e-2, rtol=2e-2) + + if current_platform.is_rocm(): # FLASH_ATTN is not supported on ROCm SLIDING_WINDOW_BACKENDS_TO_TEST = [ diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 85b7456dd8..1a41458447 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -878,12 +878,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel # must be selected statically for FULL cudagraph capture. - can_use_xqa_or_trtllm_gen_decode = ( - can_use_trtllm_attention( - self.num_qo_heads, self.num_kv_heads, is_prefill=False - ) - and not self.nvfp4_fa2 - ) + # XQA reads an NVFP4 cache only on SM12x; SM90 decodes it with fa2. + can_use_xqa_or_trtllm_gen_decode = can_use_trtllm_attention( + self.num_qo_heads, self.num_kv_heads, is_prefill=False + ) and (not self.nvfp4_fa2 or current_platform.is_device_capability_family(120)) # Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by # get_supported_kernel_block_sizes). assert self.page_size <= 64 or ( @@ -1078,7 +1076,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): ) or ( is_xqa_arch and ( - spec.kv_quant_mode.is_nvfp4 + ( + spec.kv_quant_mode.is_nvfp4 + and not current_platform.is_device_capability_family(120) + ) or not _is_xqa_head_dim_supported(spec.head_size) ) ): @@ -2283,7 +2284,6 @@ class FlashInferImpl(AttentionImpl): use_dcp = self.dcp_world_size > 1 if decode_with_xqa: assert not use_dcp - assert not self.is_kvcache_nvfp4 assert self.o_sf_scale is None assert output.dtype != FP4_DTYPE @@ -2628,7 +2628,10 @@ class FlashInferImpl(AttentionImpl): flashinfer_xqa_batch_decode_with_kv_cache( query=decode_query, - kv_cache=kv_cache_tuple, + kv_cache=( + nvfp4_kv_data if self.is_kvcache_nvfp4 else kv_cache_tuple + ), + kv_cache_sf=nvfp4_kv_block_scales, workspace_buffer=workspace_buffer, block_tables=block_tables_decode, seq_lens=seq_lens_decode,