[Attention] Extend XQA decode support on SM90 (#50439)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Andrii Skliar
OpenAI Codex
parent
fbf51c7026
commit
4b839c3378
@@ -889,7 +889,7 @@ def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
|
||||
|
||||
device = torch.device("cpu")
|
||||
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
||||
builder.use_dedicated_xqa = True
|
||||
builder.use_xqa = True
|
||||
qo_indptr = torch.tensor([0, 3, 9, 15, 15], dtype=torch.int32, device=device)
|
||||
|
||||
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
||||
@@ -905,6 +905,95 @@ def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
|
||||
assert q_cu_seq_lens.tolist() == [0, 3, 9, 15, 15]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypatch):
|
||||
"""Non-ragged XQA keeps one query/output row per padded request."""
|
||||
import unittest.mock
|
||||
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
impl = object.__new__(flashinfer_backend.FlashInferImpl)
|
||||
impl.scale = 1.0
|
||||
impl.bmm1_scale = None
|
||||
impl.bmm2_scale = None
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.is_kvcache_nvfp4 = False
|
||||
impl.head_size = 16
|
||||
impl.dcp_world_size = 1
|
||||
impl.o_sf_scale = None
|
||||
impl.window_left = -1
|
||||
impl.sinks = None
|
||||
impl.cache_config = unittest.mock.Mock()
|
||||
impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
|
||||
|
||||
layer = unittest.mock.Mock(
|
||||
_q_scale=torch.tensor(1.0),
|
||||
_q_scale_float=1.0,
|
||||
_k_scale_float=1.0,
|
||||
_v_scale_float=1.0,
|
||||
)
|
||||
decode = flashinfer_backend.FlashInferTrtllmAPIDecode(
|
||||
kernel=flashinfer_backend.FlashInferDecodeKernel.XQA,
|
||||
block_tables=torch.zeros((4, 1), dtype=torch.int32),
|
||||
seq_lens=torch.tensor([8, 8, 0, 0], dtype=torch.int32),
|
||||
max_seq_len=8,
|
||||
q_len_per_req=1,
|
||||
)
|
||||
attn_metadata = flashinfer_backend.FlashInferMetadata(
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.empty(0, dtype=torch.int64),
|
||||
q_data_type_prefill=torch.bfloat16,
|
||||
q_data_type_decode=torch.bfloat16,
|
||||
num_decodes=4,
|
||||
num_decode_tokens=2,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
causal=True,
|
||||
prefill=None,
|
||||
decode=decode,
|
||||
use_cascade=False,
|
||||
cascade_wrapper=None,
|
||||
)
|
||||
|
||||
seen_shapes = {}
|
||||
|
||||
def mock_xqa(**kwargs):
|
||||
seen_shapes["query"] = kwargs["query"].shape
|
||||
seen_shapes["out"] = kwargs["out"].shape
|
||||
seen_shapes["block_tables"] = kwargs["block_tables"].shape
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_backend,
|
||||
"_get_trtllm_workspace_buffer",
|
||||
lambda: torch.empty(1, dtype=torch.uint8),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_backend, "flashinfer_xqa_batch_decode_with_kv_cache", mock_xqa
|
||||
)
|
||||
|
||||
query = torch.zeros((4, 1, 16), dtype=torch.bfloat16)
|
||||
output = torch.empty_like(query)
|
||||
result = impl.forward(
|
||||
layer,
|
||||
query,
|
||||
torch.empty_like(query),
|
||||
torch.empty_like(query),
|
||||
torch.zeros((1, 1, 1, 32), dtype=torch.bfloat16),
|
||||
attn_metadata,
|
||||
output,
|
||||
)
|
||||
|
||||
assert result is output
|
||||
assert seen_shapes == {
|
||||
"query": torch.Size([4, 1, 16]),
|
||||
"out": torch.Size([4, 1, 16]),
|
||||
"block_tables": torch.Size([4, 1]),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
@@ -913,7 +1002,7 @@ def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
||||
builder.use_dedicated_xqa = True
|
||||
builder.use_xqa = True
|
||||
qo_indptr = torch.tensor([0, 3, 3, 3], dtype=torch.int32)
|
||||
|
||||
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
||||
@@ -929,6 +1018,42 @@ def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
|
||||
assert q_cu_seq_lens.tolist() == [0, 3, 3, 3]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("qo_indptr_values", "num_decode_tokens", "expected_q_len"),
|
||||
[
|
||||
([0, 3, 3], 3, 3),
|
||||
([0, 2, 4, 4, 4], 4, 2),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
|
||||
qo_indptr_values: list[int],
|
||||
num_decode_tokens: int,
|
||||
expected_q_len: int,
|
||||
):
|
||||
"""Padded speculative decode keeps its actual packed query width."""
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
||||
builder.use_xqa = False
|
||||
qo_indptr = torch.tensor(qo_indptr_values, dtype=torch.int32)
|
||||
|
||||
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
||||
qo_indptr,
|
||||
qo_indptr,
|
||||
num_decodes=len(qo_indptr_values) - 1,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
)
|
||||
|
||||
assert q_len == expected_q_len
|
||||
assert q_lens is None
|
||||
assert q_cu_seq_lens is not None
|
||||
assert q_cu_seq_lens.tolist() == qo_indptr_values
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
@@ -961,10 +1086,12 @@ def test_flashinfer_attention_sinks_refreshed_after_reload(dtype):
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(120)
|
||||
):
|
||||
pytest.skip("Native FlashInfer prefill with sinks requires SM12x.")
|
||||
supported = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
if not supported:
|
||||
pytest.skip("Native FlashInfer prefill with sinks requires SM90 or SM12x.")
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
|
||||
|
||||
@@ -995,7 +1122,7 @@ def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
"""FlashInfer should route supported decode through XQA and match SDPA."""
|
||||
"""FlashInfer should route SM90/SM12x decode through XQA and match SDPA."""
|
||||
supported = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
@@ -1066,16 +1193,23 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
)
|
||||
attn_metadata = builder.build(0, common_attn_metadata)
|
||||
|
||||
expected_cg_support = (
|
||||
AttentionCGSupport.UNIFORM_BATCH
|
||||
if current_platform.is_device_capability_family(120)
|
||||
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
assert (
|
||||
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
== expected_cg_support
|
||||
== AttentionCGSupport.UNIFORM_BATCH
|
||||
)
|
||||
wide_head_kv_cache_spec = FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads,
|
||||
head_size=512,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
)
|
||||
assert (
|
||||
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, wide_head_kv_cache_spec
|
||||
)
|
||||
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
assert isinstance(
|
||||
attn_metadata.decode,
|
||||
|
||||
@@ -13,6 +13,7 @@ import torch
|
||||
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.config import SpeculativeConfig, set_current_vllm_config
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
FlashInferDecodeKernel,
|
||||
@@ -73,3 +74,7 @@ def test_flashinfer_gqa_dcp_spec_decode_clamps_reorder_threshold(monkeypatch):
|
||||
builder.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN
|
||||
)
|
||||
assert builder.reorder_batch_threshold == 1
|
||||
assert (
|
||||
FlashInferMetadataBuilder.get_cudagraph_support(vllm_config, kv_cache_spec)
|
||||
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user