[Attention] Add FlashInfer XQA decode support on SM12x (#49718)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Andrii Skliar
OpenAI Codex
parent
a367cdbca0
commit
1d2d83a07f
@@ -158,6 +158,10 @@ flashinfer_trtllm_batch_decode_sparse_mla_dsv4 = _lazy_import_wrapper(
|
||||
"trtllm_batch_decode_sparse_mla_dsv4",
|
||||
fallback_fn=_missing_sparse_mla,
|
||||
)
|
||||
flashinfer_xqa_batch_decode_with_kv_cache = _lazy_import_wrapper(
|
||||
"flashinfer.decode",
|
||||
"xqa_batch_decode_with_kv_cache",
|
||||
)
|
||||
|
||||
|
||||
# Special case for autotune since it returns a context manager
|
||||
@@ -375,8 +379,8 @@ def supports_trtllm_attention(is_prefill: bool = False) -> bool:
|
||||
"""Return whether TRTLLM attention is available on the current platform
|
||||
for the given attention phase.
|
||||
|
||||
SM90 (Hopper) supports the XQA decode kernel but not TRTLLM prefill.
|
||||
SM100+ supports TRTLLM for both phases. All others are unsupported.
|
||||
SM90 (Hopper) and SM12x support the XQA decode kernel but not TRTLLM
|
||||
prefill. SM100+ supports TRTLLM for both phases. All others are unsupported.
|
||||
"""
|
||||
# Batch-invariant mode disables TRTLLM attention
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
@@ -386,8 +390,10 @@ def supports_trtllm_attention(is_prefill: bool = False) -> bool:
|
||||
if not has_nvidia_artifactory():
|
||||
return False
|
||||
|
||||
# SM90 has XQA decode; prefill is not supported.
|
||||
if current_platform.is_device_capability(90):
|
||||
# SM90 and SM12x have XQA decode only.
|
||||
if current_platform.is_device_capability(
|
||||
90
|
||||
) or current_platform.is_device_capability_family(120):
|
||||
return not is_prefill
|
||||
|
||||
# SM100/SM103 has both prefill and decode TRTLLM kernels.
|
||||
@@ -490,11 +496,11 @@ def use_trtllm_attention(
|
||||
if is_prefill:
|
||||
# Prefill auto-detection
|
||||
use_trtllm = kv_cache_dtype == "auto"
|
||||
elif current_platform.is_device_capability(90) and kv_cache_dtype.startswith(
|
||||
"fp8"
|
||||
):
|
||||
# SM90 + FP8 KV cache: prefer the XQA decode kernel. XQA does not
|
||||
# support NVFP4 KV (that is an SM100 trtllm-gen path only).
|
||||
elif (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
) and kv_cache_dtype.startswith("fp8"):
|
||||
# SM90/SM12x + FP8 KV cache: prefer the XQA decode kernel.
|
||||
use_trtllm = True
|
||||
else:
|
||||
# Decode auto-detection
|
||||
@@ -1036,6 +1042,7 @@ __all__ = [
|
||||
"trtllm_fp4_block_scale_moe",
|
||||
"flashinfer_trtllm_batch_decode_with_kv_cache_mla",
|
||||
"flashinfer_trtllm_batch_decode_sparse_mla_dsv4",
|
||||
"flashinfer_xqa_batch_decode_with_kv_cache",
|
||||
"autotune",
|
||||
"has_flashinfer_moe",
|
||||
"has_flashinfer_comm",
|
||||
|
||||
Reference in New Issue
Block a user