[Attention] Add FlashInfer XQA decode support on SM12x (#49718)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Andrii Skliar
OpenAI Codex
parent
a367cdbca0
commit
1d2d83a07f
@@ -101,6 +101,20 @@ def test_supports_sm90_decode_only(_art, _family, _cap):
|
||||
assert supports_trtllm_attention(is_prefill=True) is False
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability", return_value=False
|
||||
)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
side_effect=lambda capability: capability == 120,
|
||||
)
|
||||
@patch("vllm.utils.flashinfer.has_nvidia_artifactory", return_value=True)
|
||||
def test_supports_sm12x_decode_only(_art, _family, _cap):
|
||||
assert supports_trtllm_attention(is_prefill=False) is True
|
||||
assert supports_trtllm_attention(is_prefill=True) is False
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
|
||||
@@ -250,6 +250,7 @@ def run_attention_backend(
|
||||
attn_type: AttentionType = AttentionType.DECODER,
|
||||
sliding_window: int | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
sinks: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run attention computation using the specified backend's AttentionImpl."""
|
||||
|
||||
@@ -277,6 +278,7 @@ def run_attention_backend(
|
||||
window_left=-1, # No sliding window
|
||||
logits_soft_cap=0.0, # No soft cap
|
||||
sm_scale=1.0 / (head_size**0.5), # Standard scale
|
||||
has_sinks=sinks is not None,
|
||||
)
|
||||
for layer_name in layer_names
|
||||
}
|
||||
@@ -318,6 +320,7 @@ def run_attention_backend(
|
||||
sliding_window=sliding_window,
|
||||
attn_type=attn_type,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
**({"sinks": sinks} if sinks is not None else {}),
|
||||
)
|
||||
|
||||
# Create mock layer and output buffer
|
||||
@@ -354,10 +357,11 @@ def _test_backend_correctness(
|
||||
rtol: float = 1e-2,
|
||||
tensor_parallel_size: int = 1,
|
||||
kv_cache_dtype: str = "auto",
|
||||
use_sinks: bool = False,
|
||||
):
|
||||
"""
|
||||
Test that all backends produce similar outputs to a reference implementation
|
||||
using torch.nn.functional.scaled_dot_product_attention.
|
||||
using FlexAttention or an explicit attention-sink reference.
|
||||
|
||||
This test works by:
|
||||
1. Generating a batch of sequences with specified context and query lengths.
|
||||
@@ -417,6 +421,11 @@ def _test_backend_correctness(
|
||||
num_kv_heads = vllm_config.model_config.get_num_kv_heads(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
sinks = (
|
||||
torch.linspace(-1.0, 1.0, num_q_heads, dtype=torch.float32, device=device)
|
||||
if use_sinks
|
||||
else None
|
||||
)
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
sliding_window = vllm_config.model_config.get_sliding_window()
|
||||
dtype = _convert_dtype_to_torch(vllm_config.model_config.dtype)
|
||||
@@ -471,17 +480,38 @@ def _test_backend_correctness(
|
||||
kv_len = s_len
|
||||
|
||||
final_mask_mod = partial(mask_mod, context_len=context_len)
|
||||
block_mask = create_block_mask(
|
||||
final_mask_mod, B=None, H=None, Q_LEN=q_len, KV_LEN=kv_len, device=device
|
||||
)
|
||||
sdpa_out_i = flex_attention(
|
||||
q_sdpa_in,
|
||||
k_sdpa_in,
|
||||
v_sdpa_in,
|
||||
block_mask=block_mask,
|
||||
scale=scale,
|
||||
enable_gqa=True,
|
||||
)
|
||||
if sinks is None:
|
||||
block_mask = create_block_mask(
|
||||
final_mask_mod,
|
||||
B=None,
|
||||
H=None,
|
||||
Q_LEN=q_len,
|
||||
KV_LEN=kv_len,
|
||||
device=device,
|
||||
)
|
||||
sdpa_out_i = flex_attention(
|
||||
q_sdpa_in,
|
||||
k_sdpa_in,
|
||||
v_sdpa_in,
|
||||
block_mask=block_mask,
|
||||
scale=scale,
|
||||
enable_gqa=True,
|
||||
)
|
||||
else:
|
||||
scores = (
|
||||
torch.matmul(q_sdpa_in.float(), k_sdpa_in.float().transpose(-2, -1))
|
||||
* scale
|
||||
)
|
||||
q_idx = torch.arange(q_len, device=device).unsqueeze(1)
|
||||
kv_idx = torch.arange(kv_len, device=device).unsqueeze(0)
|
||||
zero = torch.zeros((), dtype=torch.int32, device=device)
|
||||
valid = final_mask_mod(zero, zero, q_idx, kv_idx)
|
||||
scores.masked_fill_(~valid.unsqueeze(0).unsqueeze(0), -torch.inf)
|
||||
sink_logits = sinks.view(1, num_q_heads, 1, 1).expand(1, -1, q_len, -1)
|
||||
weights = torch.softmax(torch.cat((scores, sink_logits), dim=-1), dim=-1)[
|
||||
..., :kv_len
|
||||
]
|
||||
sdpa_out_i = torch.matmul(weights, v_sdpa_in.float()).to(dtype)
|
||||
|
||||
all_sdpa_outputs.append(sdpa_out_i.transpose(1, 2).squeeze(0))
|
||||
|
||||
@@ -574,6 +604,7 @@ def _test_backend_correctness(
|
||||
sliding_window=sliding_window,
|
||||
attn_type=attn_type,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
sinks=sinks,
|
||||
)
|
||||
finally:
|
||||
if reset_kv_cache_layout:
|
||||
@@ -706,6 +737,86 @@ def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype():
|
||||
assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_draft_masks():
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
device = torch.device("cpu")
|
||||
causal = flashinfer_backend._make_xqa_draft_block_mask(3, True, device)
|
||||
full = flashinfer_backend._make_xqa_draft_block_mask(3, False, device)
|
||||
ragged = flashinfer_backend._make_xqa_ragged_draft_block_mask(
|
||||
[2, 3], 3, True, device
|
||||
)
|
||||
|
||||
assert torch.equal(
|
||||
causal.view(torch.int16),
|
||||
torch.tensor([[1, 0], [3, 0], [7, 0]], dtype=torch.int16),
|
||||
)
|
||||
assert torch.equal(
|
||||
full.view(torch.int16), torch.tensor([[7, 0]] * 3, dtype=torch.int16)
|
||||
)
|
||||
assert torch.equal(
|
||||
ragged.view(torch.int16),
|
||||
torch.tensor(
|
||||
[[1, 0], [3, 0], [1, 0], [3, 0], [7, 0]],
|
||||
dtype=torch.int16,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
|
||||
"""CUDA-graph padding stays as zero-length requests in ragged offsets."""
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
device = torch.device("cpu")
|
||||
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
||||
builder.use_dedicated_xqa = True
|
||||
qo_indptr = torch.tensor([0, 3, 9, 15, 15], dtype=torch.int32, device=device)
|
||||
|
||||
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
||||
qo_indptr,
|
||||
qo_indptr,
|
||||
num_decodes=4,
|
||||
num_decode_tokens=21,
|
||||
)
|
||||
|
||||
assert q_len == 6
|
||||
assert q_lens == [3, 6, 6, 0]
|
||||
assert q_cu_seq_lens is not None
|
||||
assert q_cu_seq_lens.tolist() == [0, 3, 9, 15, 15]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
||||
builder.use_dedicated_xqa = True
|
||||
qo_indptr = torch.tensor([0, 3, 3, 3], dtype=torch.int32)
|
||||
|
||||
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
||||
qo_indptr,
|
||||
qo_indptr,
|
||||
num_decodes=3,
|
||||
num_decode_tokens=6,
|
||||
)
|
||||
|
||||
assert q_len == 3
|
||||
assert q_lens == [3, 0, 0]
|
||||
assert q_cu_seq_lens is not None
|
||||
assert q_cu_seq_lens.tolist() == [0, 3, 3, 3]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
@@ -737,10 +848,48 @@ def test_flashinfer_attention_sinks_refreshed_after_reload(dtype):
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_sm90_xqa_decode_correctness(default_vllm_config):
|
||||
"""FlashInfer should route Hopper decode through XQA and match SDPA."""
|
||||
if not current_platform.is_cuda() or not current_platform.is_device_capability(90):
|
||||
pytest.skip("FlashInfer XQA decode requires SM90.")
|
||||
def test_flashinfer_native_prefill_with_sinks():
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(120)
|
||||
):
|
||||
pytest.skip("Native FlashInfer prefill with sinks requires SM12x.")
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
|
||||
|
||||
if not FlashInferBackend.supports_sink():
|
||||
pytest.skip("FlashInfer attention sinks are not available in this setup.")
|
||||
|
||||
def causal_mask_mod(
|
||||
b: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
q_idx: torch.Tensor,
|
||||
kv_idx: torch.Tensor,
|
||||
*,
|
||||
context_len: int,
|
||||
):
|
||||
return (q_idx + context_len) >= kv_idx
|
||||
|
||||
_test_backend_correctness(
|
||||
BATCH_SPECS["small_prefill"],
|
||||
"meta-llama/Meta-Llama-3-8B",
|
||||
[AttentionBackendEnum.FLASHINFER],
|
||||
causal_mask_mod,
|
||||
use_sinks=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
"""FlashInfer should route supported decode through XQA and match SDPA."""
|
||||
supported = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
if not supported:
|
||||
pytest.skip("FlashInfer XQA decode requires SM90 or SM12x.")
|
||||
|
||||
import unittest.mock
|
||||
|
||||
@@ -805,11 +954,16 @@ def test_flashinfer_sm90_xqa_decode_correctness(default_vllm_config):
|
||||
)
|
||||
attn_metadata = builder.build(0, common_attn_metadata)
|
||||
|
||||
expected_cg_support = (
|
||||
AttentionCGSupport.UNIFORM_BATCH
|
||||
if current_platform.is_device_capability_family(120)
|
||||
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
assert (
|
||||
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
== expected_cg_support
|
||||
)
|
||||
assert isinstance(
|
||||
attn_metadata.decode,
|
||||
|
||||
Reference in New Issue
Block a user