[Attention] Decode an NVFP4 KV cache with XQA on SM12x (#60452)

Signed-off-by: lesj0610 <lesj0610@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
SeongJun Lee
2026-10-09 22:24:00 -06:00
committed by GitHub
co-authored by Misha Goin
parent 29e4d2d42e
commit 9bd074a996
3 changed files with 121 additions and 10 deletions
+14
View File
@@ -62,3 +62,17 @@ steps:
commands:
- pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":nvidia: (DGX) Spark FlashInfer NVFP4 XQA"
key: flashinfer-nvfp4-xqa-dgx-spark
timeout_in_minutes: 30
device: dgx-spark
optional: true
num_devices: 1
depends_on:
- arm64-image-build
source_file_dependencies:
- vllm/v1/attention/backends/flashinfer.py
- tests/v1/attention/test_attention_backends.py
commands:
- pytest -v -s v1/attention/test_attention_backends.py -k test_flashinfer_xqa_nvfp4_decode_correctness
+95 -1
View File
@@ -31,6 +31,7 @@ from vllm.utils.torch_utils import (
STR_DTYPE_TO_TORCH_DTYPE,
is_quantized_kv_cache,
is_torch_equal_or_newer,
nvfp4_kv_cache_full_dim,
set_random_seed,
)
from vllm.v1.attention.backend import (
@@ -41,7 +42,7 @@ from vllm.v1.attention.backend import (
max_decode_query_len,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode
BACKENDS_TO_TEST = [
AttentionBackendEnum.FLASH_ATTN,
@@ -1698,6 +1699,99 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
or not current_platform.is_device_capability_family(120),
reason="NVFP4 XQA decode requires SM12x.",
)
@pytest.mark.parametrize("use_cuda_graph", [False, True])
@pytest.mark.parametrize("q_len", [1, 4])
def test_flashinfer_xqa_nvfp4_decode_correctness(
default_vllm_config, q_len, use_cuda_graph
):
"""Single-token and spec-decode queries over an NVFP4 cache decode with
XQA, eager and replayed, and match the fa2 prefill that wrote the cache."""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as fi
seq_lens, bs, d = [40, 300, 1027], 16, 256
config = create_vllm_config(max_model_len=max(seq_lens), block_size=bs)
# GQA 6 at head_dim 256, as Qwen3.8-27B at TP=2.
config.model_config.model_arch_config = replace(
config.model_config.model_arch_config,
total_num_attention_heads=24,
total_num_kv_heads=4,
head_size=d,
)
config.cache_config.cache_dtype = "nvfp4"
if q_len > 1:
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=q_len - 1
)
spec = FullAttentionSpec(
block_size=bs,
num_kv_heads=4,
head_size=d,
dtype=torch.uint8,
kv_quant_mode=KVQuantMode.NVFP4,
)
cg_support = fi.FlashInferMetadataBuilder.get_cudagraph_support(config, spec)
assert cg_support == AttentionCGSupport.UNIFORM_BATCH
device = torch.device(f"{DEVICE_TYPE}:0")
# Request i owns consecutive pages from starts[i]; page 0 stays null.
pages = [0] + [cdiv(s, bs) for s in seq_lens[:-1]]
starts = 1 + torch.tensor(pages).cumsum(0)
max_pages = cdiv(max(seq_lens), bs)
block_table = (starts[:, None] + torch.arange(max_pages)).int().to(device)
full_dim = nvfp4_kv_cache_full_dim(d)
kv_cache = torch.zeros(
int(starts[-1]) + max_pages, bs, 8, full_dim, dtype=torch.uint8, device=device
).transpose(1, 2)
q, k, v = (
torch.randn(sum(seq_lens), h, d).bfloat16().to(device) for h in (24, 4, 4)
)
def run(query_lens, use_cuda_graph=False):
req = torch.repeat_interleave(torch.arange(3), torch.tensor(query_lens))
pos = torch.cat([torch.arange(s - n, s) for s, n in zip(seq_lens, query_lens)])
rows = pos + torch.tensor([0, *seq_lens]).cumsum(0)[req]
metadata = replace(
create_common_attn_metadata(BatchSpec(seq_lens, query_lens), bs, device),
block_table_tensor=block_table,
slot_mapping=(starts[req] * bs + pos).to(device),
)
with set_current_vllm_config(config):
return rows, run_attention_backend(
AttentionBackendEnum.FLASHINFER,
spec,
["placeholder"],
config,
device,
metadata,
q[rows],
k[rows],
v[rows],
kv_cache,
kv_cache_dtype="nvfp4",
use_cuda_graph=use_cuda_graph,
layer_k_scale=0.02,
layer_v_scale=0.03,
)
# The full prefill writes every token through the NVFP4 writer; its fa2
# output for the last q_len tokens of each request is the reference.
_, reference = run(seq_lens)
xqa_fn = fi.flashinfer_xqa_batch_decode_with_kv_cache
with unittest.mock.patch.object(
fi, "flashinfer_xqa_batch_decode_with_kv_cache", wraps=xqa_fn
) as xqa:
rows, output = run([q_len] * len(seq_lens), use_cuda_graph)
assert xqa.called
torch.testing.assert_close(output, reference[rows], atol=2e-2, rtol=2e-2)
if current_platform.is_rocm():
# FLASH_ATTN is not supported on ROCm
SLIDING_WINDOW_BACKENDS_TO_TEST = [
+12 -9
View File
@@ -878,12 +878,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel
# must be selected statically for FULL cudagraph capture.
can_use_xqa_or_trtllm_gen_decode = (
can_use_trtllm_attention(
self.num_qo_heads, self.num_kv_heads, is_prefill=False
)
and not self.nvfp4_fa2
)
# XQA reads an NVFP4 cache only on SM12x; SM90 decodes it with fa2.
can_use_xqa_or_trtllm_gen_decode = can_use_trtllm_attention(
self.num_qo_heads, self.num_kv_heads, is_prefill=False
) and (not self.nvfp4_fa2 or current_platform.is_device_capability_family(120))
# Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by
# get_supported_kernel_block_sizes).
assert self.page_size <= 64 or (
@@ -1078,7 +1076,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
) or (
is_xqa_arch
and (
spec.kv_quant_mode.is_nvfp4
(
spec.kv_quant_mode.is_nvfp4
and not current_platform.is_device_capability_family(120)
)
or not _is_xqa_head_dim_supported(spec.head_size)
)
):
@@ -2283,7 +2284,6 @@ class FlashInferImpl(AttentionImpl):
use_dcp = self.dcp_world_size > 1
if decode_with_xqa:
assert not use_dcp
assert not self.is_kvcache_nvfp4
assert self.o_sf_scale is None
assert output.dtype != FP4_DTYPE
@@ -2628,7 +2628,10 @@ class FlashInferImpl(AttentionImpl):
flashinfer_xqa_batch_decode_with_kv_cache(
query=decode_query,
kv_cache=kv_cache_tuple,
kv_cache=(
nvfp4_kv_data if self.is_kvcache_nvfp4 else kv_cache_tuple
),
kv_cache_sf=nvfp4_kv_block_scales,
workspace_buffer=workspace_buffer,
block_tables=block_tables_decode,
seq_lens=seq_lens_decode,