Xqa decode kernels (#43232)

Signed-off-by: Dan Blanaru <48605845+DanBlanaru@users.noreply.github.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Dan Blanaru
2026-07-02 12:32:05 -07:00
committed by GitHub
co-authored by Matthew Bonanni
parent d715b3aa1e
commit d29125c085
7 changed files with 538 additions and 143 deletions
@@ -78,14 +78,29 @@ def test_supports_sm100_with_artifactory(_art, _cap):
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
@patch(
"vllm.utils.flashinfer.current_platform.is_device_capability", return_value=False
)
@patch(
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
return_value=False,
)
def test_supports_non_sm100_platform(_cap):
def test_supports_unsupported_platform(_family, _cap):
assert supports_trtllm_attention() is False
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
@patch("vllm.utils.flashinfer.current_platform.is_device_capability", return_value=True)
@patch(
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
return_value=False,
)
@patch("vllm.utils.flashinfer.has_nvidia_artifactory", return_value=True)
def test_supports_sm90_decode_only(_art, _family, _cap):
assert supports_trtllm_attention(is_prefill=False) is True
assert supports_trtllm_attention(is_prefill=True) is False
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
@patch(
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
+120 -4
View File
@@ -16,7 +16,7 @@ from tests.v1.attention.utils import (
try_backend_includes_kv_cache_update,
try_get_attention_backend,
)
from vllm.config import ModelConfig
from vllm.config import ModelConfig, set_current_vllm_config
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
@@ -24,7 +24,11 @@ from vllm.utils.torch_utils import (
is_torch_equal_or_newer,
set_random_seed,
)
from vllm.v1.attention.backend import AttentionType, CommonAttentionMetadata
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionType,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.utils import (
set_kv_cache_layout,
@@ -626,6 +630,118 @@ def test_causal_backend_correctness(
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype():
"""XQA decode should only apply q_scale when decode Q is FP8."""
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
class MockLayer:
_q_scale_float = 2.0
_k_scale_float = 3.0
impl = object.__new__(flashinfer_backend.FlashInferImpl)
impl.scale = 0.5
impl.kv_cache_dtype = "fp8"
assert impl.get_xqa_bmm1_scale(MockLayer, torch.bfloat16) == 1.5
assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_sm90_xqa_decode_correctness(default_vllm_config):
"""FlashInfer should route Hopper decode through XQA and match SDPA."""
if not current_platform.is_cuda() or not current_platform.is_device_capability(90):
pytest.skip("FlashInfer XQA decode requires SM90.")
import unittest.mock
from vllm.utils.flashinfer import can_use_trtllm_attention
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
from vllm.v1.attention.backends.utils import PerLayerParameters
def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
return {
"placeholder": PerLayerParameters(
window_left=-1,
logits_soft_cap=0.0,
sm_scale=1.0,
)
}
def causal_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
):
return (q_idx + context_len) >= kv_idx
batch_spec = BATCH_SPECS["small_decode"]
vllm_config = create_vllm_config(
model_name="meta-llama/Meta-Llama-3-8B",
max_model_len=max(batch_spec.seq_lens),
block_size=16,
)
device = torch.device(f"{DEVICE_TYPE}:0")
kv_cache_spec = FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=vllm_config.model_config.get_num_kv_heads(
vllm_config.parallel_config
),
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
)
with set_current_vllm_config(vllm_config):
if not can_use_trtllm_attention(
vllm_config.model_config.get_num_attention_heads(
vllm_config.parallel_config
),
kv_cache_spec.num_kv_heads,
is_prefill=False,
):
pytest.skip("FlashInfer XQA decode is not available in this setup.")
with unittest.mock.patch(
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
mock_get_per_layer_parameters,
):
builder = flashinfer_backend.FlashInferMetadataBuilder(
kv_cache_spec, ["placeholder"], vllm_config, device
)
common_attn_metadata = create_common_attn_metadata(
batch_spec, vllm_config.cache_config.block_size, device
)
attn_metadata = builder.build(0, common_attn_metadata)
assert (
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
vllm_config, kv_cache_spec
)
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
assert isinstance(
attn_metadata.decode,
flashinfer_backend.FlashInferTrtllmAPIDecode,
)
assert attn_metadata.decode.kernel == flashinfer_backend.FlashInferDecodeKernel.XQA
_test_backend_correctness(
batch_spec,
"meta-llama/Meta-Llama-3-8B",
[AttentionBackendEnum.FLASHINFER],
causal_mask_mod,
)
if current_platform.is_rocm():
# FLASH_ATTN is not supported on ROCm
SLIDING_WINDOW_BACKENDS_TO_TEST = [
@@ -656,7 +772,7 @@ else:
@pytest.mark.parametrize("model", ["microsoft/Phi-tiny-MoE-instruct"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
def test_sliding_window_backend_correctness(
batch_spec_name: str, model: str, tensor_parallel_size: int
default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int
):
"""Test backend's correctness with sliding window attention."""
@@ -718,7 +834,7 @@ def test_sliding_window_backend_correctness(
@pytest.mark.parametrize("model", ["google/embeddinggemma-300m"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2])
def test_sliding_window_encoder_backend_correctness(
batch_spec_name: str, model: str, tensor_parallel_size: int
default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int
):
"""Test backend's correctness with sliding window attention."""
@@ -32,9 +32,10 @@ if not current_platform.is_device_capability_family(100):
)
from vllm.v1.attention.backends.flashinfer import ( # noqa: E402
FlashInferDecodeKernel,
FlashInferImpl,
FlashInferMetadataBuilder,
TRTLLMDecode,
FlashInferTrtllmAPIDecode,
TRTLLMPrefill,
)
@@ -435,9 +436,11 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
f"Expected TRTLLMPrefill, got {type(attn_metadata.prefill)}"
)
if has_decodes:
assert isinstance(attn_metadata.decode, TRTLLMDecode), (
f"Expected TRTLLMDecode, got {type(attn_metadata.decode)}"
assert isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode), (
"Expected FlashInferTrtllmAPIDecode, got "
f"{type(attn_metadata.decode)}"
)
assert attn_metadata.decode.kernel == FlashInferDecodeKernel.TRTLLM_GEN
impl = FlashInferImpl(
num_heads=num_q_heads,