[Attention] Decode an NVFP4 KV cache with XQA on SM12x (#60452)

Signed-off-by: lesj0610 <lesj0610@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
SeongJun Lee
2026-10-09 22:24:00 -06:00
committed by GitHub
co-authored by Misha Goin
parent 29e4d2d42e
commit 9bd074a996
3 changed files with 121 additions and 10 deletions
+95 -1
View File
@@ -31,6 +31,7 @@ from vllm.utils.torch_utils import (
STR_DTYPE_TO_TORCH_DTYPE,
is_quantized_kv_cache,
is_torch_equal_or_newer,
nvfp4_kv_cache_full_dim,
set_random_seed,
)
from vllm.v1.attention.backend import (
@@ -41,7 +42,7 @@ from vllm.v1.attention.backend import (
max_decode_query_len,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode
BACKENDS_TO_TEST = [
AttentionBackendEnum.FLASH_ATTN,
@@ -1698,6 +1699,99 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
or not current_platform.is_device_capability_family(120),
reason="NVFP4 XQA decode requires SM12x.",
)
@pytest.mark.parametrize("use_cuda_graph", [False, True])
@pytest.mark.parametrize("q_len", [1, 4])
def test_flashinfer_xqa_nvfp4_decode_correctness(
default_vllm_config, q_len, use_cuda_graph
):
"""Single-token and spec-decode queries over an NVFP4 cache decode with
XQA, eager and replayed, and match the fa2 prefill that wrote the cache."""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as fi
seq_lens, bs, d = [40, 300, 1027], 16, 256
config = create_vllm_config(max_model_len=max(seq_lens), block_size=bs)
# GQA 6 at head_dim 256, as Qwen3.8-27B at TP=2.
config.model_config.model_arch_config = replace(
config.model_config.model_arch_config,
total_num_attention_heads=24,
total_num_kv_heads=4,
head_size=d,
)
config.cache_config.cache_dtype = "nvfp4"
if q_len > 1:
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=q_len - 1
)
spec = FullAttentionSpec(
block_size=bs,
num_kv_heads=4,
head_size=d,
dtype=torch.uint8,
kv_quant_mode=KVQuantMode.NVFP4,
)
cg_support = fi.FlashInferMetadataBuilder.get_cudagraph_support(config, spec)
assert cg_support == AttentionCGSupport.UNIFORM_BATCH
device = torch.device(f"{DEVICE_TYPE}:0")
# Request i owns consecutive pages from starts[i]; page 0 stays null.
pages = [0] + [cdiv(s, bs) for s in seq_lens[:-1]]
starts = 1 + torch.tensor(pages).cumsum(0)
max_pages = cdiv(max(seq_lens), bs)
block_table = (starts[:, None] + torch.arange(max_pages)).int().to(device)
full_dim = nvfp4_kv_cache_full_dim(d)
kv_cache = torch.zeros(
int(starts[-1]) + max_pages, bs, 8, full_dim, dtype=torch.uint8, device=device
).transpose(1, 2)
q, k, v = (
torch.randn(sum(seq_lens), h, d).bfloat16().to(device) for h in (24, 4, 4)
)
def run(query_lens, use_cuda_graph=False):
req = torch.repeat_interleave(torch.arange(3), torch.tensor(query_lens))
pos = torch.cat([torch.arange(s - n, s) for s, n in zip(seq_lens, query_lens)])
rows = pos + torch.tensor([0, *seq_lens]).cumsum(0)[req]
metadata = replace(
create_common_attn_metadata(BatchSpec(seq_lens, query_lens), bs, device),
block_table_tensor=block_table,
slot_mapping=(starts[req] * bs + pos).to(device),
)
with set_current_vllm_config(config):
return rows, run_attention_backend(
AttentionBackendEnum.FLASHINFER,
spec,
["placeholder"],
config,
device,
metadata,
q[rows],
k[rows],
v[rows],
kv_cache,
kv_cache_dtype="nvfp4",
use_cuda_graph=use_cuda_graph,
layer_k_scale=0.02,
layer_v_scale=0.03,
)
# The full prefill writes every token through the NVFP4 writer; its fa2
# output for the last q_len tokens of each request is the reference.
_, reference = run(seq_lens)
xqa_fn = fi.flashinfer_xqa_batch_decode_with_kv_cache
with unittest.mock.patch.object(
fi, "flashinfer_xqa_batch_decode_with_kv_cache", wraps=xqa_fn
) as xqa:
rows, output = run([q_len] * len(seq_lens), use_cuda_graph)
assert xqa.called
torch.testing.assert_close(output, reference[rows], atol=2e-2, rtol=2e-2)
if current_platform.is_rocm():
# FLASH_ATTN is not supported on ROCm
SLIDING_WINDOW_BACKENDS_TO_TEST = [