[Attention] Add FlashInfer XQA decode support on SM12x (#49718)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andrii Skliar
2026-08-11 16:32:49 -07:00
committed by GitHub
co-authored by Andrii Skliar OpenAI Codex
parent a367cdbca0
commit 1d2d83a07f
4 changed files with 413 additions and 57 deletions
+16 -9
View File
@@ -158,6 +158,10 @@ flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
"trtllm_batch_decode_sparse_mla_dsv4",
fallback_fn=_missing_sparse_mla,
)
flashinfer_xqa_batch_decode_with_kv_cache = _lazy_import_wrapper(
"flashinfer.decode",
"xqa_batch_decode_with_kv_cache",
)
# Special case for autotune since it returns a context manager
@@ -375,8 +379,8 @@ def supports_trtllm_attention(is_prefill: bool = False) -> bool:
"""Return whether TRTLLM attention is available on the current platform
for the given attention phase.
SM90 (Hopper) supports the XQA decode kernel but not TRTLLM prefill.
SM100+ supports TRTLLM for both phases. All others are unsupported.
SM90 (Hopper) and SM12x support the XQA decode kernel but not TRTLLM
prefill. SM100+ supports TRTLLM for both phases. All others are unsupported.
"""
# Batch-invariant mode disables TRTLLM attention
if envs.VLLM_BATCH_INVARIANT:
@@ -386,8 +390,10 @@ def supports_trtllm_attention(is_prefill: bool = False) -> bool:
if not has_nvidia_artifactory():
return False
# SM90 has XQA decode; prefill is not supported.
if current_platform.is_device_capability(90):
# SM90 and SM12x have XQA decode only.
if current_platform.is_device_capability(
90
) or current_platform.is_device_capability_family(120):
return not is_prefill
# SM100/SM103 has both prefill and decode TRTLLM kernels.
@@ -490,11 +496,11 @@ def use_trtllm_attention(
if is_prefill:
# Prefill auto-detection
use_trtllm = kv_cache_dtype == "auto"
elif current_platform.is_device_capability(90) and kv_cache_dtype.startswith(
"fp8"
):
# SM90 + FP8 KV cache: prefer the XQA decode kernel. XQA does not
# support NVFP4 KV (that is an SM100 trtllm-gen path only).
elif (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(120)
) and kv_cache_dtype.startswith("fp8"):
# SM90/SM12x + FP8 KV cache: prefer the XQA decode kernel.
use_trtllm = True
else:
# Decode auto-detection
@@ -1036,6 +1042,7 @@ __all__ = [
"trtllm_fp4_block_scale_moe",
"flashinfer_trtllm_batch_decode_with_kv_cache_mla",
"flashinfer_trtllm_batch_decode_sparse_mla_dsv4",
"flashinfer_xqa_batch_decode_with_kv_cache",
"autotune",
"has_flashinfer_moe",
"has_flashinfer_comm",